View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.io.PrintWriter;
24  import java.io.StringWriter;
25  import java.net.URI;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.Collections;
30  import java.util.Comparator;
31  import java.util.HashMap;
32  import java.util.HashSet;
33  import java.util.Iterator;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Map.Entry;
37  import java.util.Set;
38  import java.util.SortedMap;
39  import java.util.SortedSet;
40  import java.util.TreeMap;
41  import java.util.TreeSet;
42  import java.util.concurrent.Callable;
43  import java.util.concurrent.ConcurrentSkipListMap;
44  import java.util.concurrent.ExecutionException;
45  import java.util.concurrent.ExecutorService;
46  import java.util.concurrent.Future;
47  import java.util.concurrent.ScheduledThreadPoolExecutor;
48  import java.util.concurrent.atomic.AtomicInteger;
49  
50  import org.apache.commons.logging.Log;
51  import org.apache.commons.logging.LogFactory;
52  import org.apache.hadoop.classification.InterfaceAudience;
53  import org.apache.hadoop.classification.InterfaceStability;
54  import org.apache.hadoop.conf.Configuration;
55  import org.apache.hadoop.conf.Configured;
56  import org.apache.hadoop.fs.FileStatus;
57  import org.apache.hadoop.fs.FileSystem;
58  import org.apache.hadoop.fs.Path;
59  import org.apache.hadoop.fs.permission.FsAction;
60  import org.apache.hadoop.hbase.Abortable;
61  import org.apache.hadoop.hbase.Cell;
62  import org.apache.hadoop.hbase.ClusterStatus;
63  import org.apache.hadoop.hbase.HBaseConfiguration;
64  import org.apache.hadoop.hbase.HColumnDescriptor;
65  import org.apache.hadoop.hbase.HConstants;
66  import org.apache.hadoop.hbase.HRegionInfo;
67  import org.apache.hadoop.hbase.HRegionLocation;
68  import org.apache.hadoop.hbase.HTableDescriptor;
69  import org.apache.hadoop.hbase.KeyValue;
70  import org.apache.hadoop.hbase.MasterNotRunningException;
71  import org.apache.hadoop.hbase.ServerName;
72  import org.apache.hadoop.hbase.TableName;
73  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
74  import org.apache.hadoop.hbase.catalog.MetaEditor;
75  import org.apache.hadoop.hbase.client.Delete;
76  import org.apache.hadoop.hbase.client.Get;
77  import org.apache.hadoop.hbase.client.HBaseAdmin;
78  import org.apache.hadoop.hbase.client.HConnectable;
79  import org.apache.hadoop.hbase.client.HConnection;
80  import org.apache.hadoop.hbase.client.HConnectionManager;
81  import org.apache.hadoop.hbase.client.HTable;
82  import org.apache.hadoop.hbase.client.MetaScanner;
83  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
84  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
85  import org.apache.hadoop.hbase.client.Put;
86  import org.apache.hadoop.hbase.client.Result;
87  import org.apache.hadoop.hbase.client.RowMutations;
88  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
89  import org.apache.hadoop.hbase.io.hfile.HFile;
90  import org.apache.hadoop.hbase.master.MasterFileSystem;
91  import org.apache.hadoop.hbase.master.RegionState;
92  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
93  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
94  import org.apache.hadoop.hbase.regionserver.HRegion;
95  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
96  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
97  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
98  import org.apache.hadoop.hbase.security.UserProvider;
99  import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
100 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
101 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
102 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
103 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
104 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
105 import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
106 import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
107 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
108 import org.apache.hadoop.io.IOUtils;
109 import org.apache.hadoop.security.AccessControlException;
110 import org.apache.hadoop.security.UserGroupInformation;
111 import org.apache.hadoop.util.ReflectionUtils;
112 import org.apache.hadoop.util.Tool;
113 import org.apache.hadoop.util.ToolRunner;
114 import org.apache.zookeeper.KeeperException;
115 
116 import com.google.common.base.Joiner;
117 import com.google.common.base.Preconditions;
118 import com.google.common.collect.Lists;
119 import com.google.common.collect.Multimap;
120 import com.google.common.collect.TreeMultimap;
121 import com.google.protobuf.ServiceException;
122 
123 /**
124  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
125  * table integrity problems in a corrupted HBase.
126  * <p>
127  * Region consistency checks verify that hbase:meta, region deployment on region
128  * servers and the state of data in HDFS (.regioninfo files) all are in
129  * accordance.
130  * <p>
131  * Table integrity checks verify that all possible row keys resolve to exactly
132  * one region of a table.  This means there are no individual degenerate
133  * or backwards regions; no holes between regions; and that there are no
134  * overlapping regions.
135  * <p>
136  * The general repair strategy works in two phases:
137  * <ol>
138  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
139  * <li> Repair Region Consistency with hbase:meta and assignments
140  * </ol>
141  * <p>
142  * For table integrity repairs, the tables' region directories are scanned
143  * for .regioninfo files.  Each table's integrity is then verified.  If there
144  * are any orphan regions (regions with no .regioninfo files) or holes, new
145  * regions are fabricated.  Backwards regions are sidelined as well as empty
146  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
147  * a new region is created and all data is merged into the new region.
148  * <p>
149  * Table integrity repairs deal solely with HDFS and could potentially be done
150  * offline -- the hbase region servers or master do not need to be running.
151  * This phase can eventually be used to completely reconstruct the hbase:meta table in
152  * an offline fashion.
153  * <p>
154  * Region consistency requires three conditions -- 1) valid .regioninfo file
155  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
156  * and 3) a region is deployed only at the regionserver that was assigned to
157  * with proper state in the master.
158  * <p>
159  * Region consistency repairs require hbase to be online so that hbck can
160  * contact the HBase master and region servers.  The hbck#connect() method must
161  * first be called successfully.  Much of the region consistency information
162  * is transient and less risky to repair.
163  * <p>
164  * If hbck is run from the command line, there are a handful of arguments that
165  * can be used to limit the kinds of repairs hbck will do.  See the code in
166  * {@link #printUsageAndExit()} for more details.
167  */
168 @InterfaceAudience.Public
169 @InterfaceStability.Evolving
170 public class HBaseFsck extends Configured {
171   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
172   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
173   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
174   private static boolean rsSupportsOffline = true;
175   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
176   private static final int DEFAULT_MAX_MERGE = 5;
177   private static final String TO_BE_LOADED = "to_be_loaded";
178 
179   /**********************
180    * Internal resources
181    **********************/
182   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
183   private ClusterStatus status;
184   private HConnection connection;
185   private HBaseAdmin admin;
186   private HTable meta;
187   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
188   protected ExecutorService executor;
189   private long startMillis = System.currentTimeMillis();
190   private HFileCorruptionChecker hfcc;
191   private int retcode = 0;
192 
193   /***********
194    * Options
195    ***********/
196   private static boolean details = false; // do we display the full report
197   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
198   private boolean fixAssignments = false; // fix assignment errors?
199   private boolean fixMeta = false; // fix meta errors?
200   private boolean checkHdfs = true; // load and check fs consistency?
201   private boolean fixHdfsHoles = false; // fix fs holes?
202   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
203   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
204   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
205   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
206   private boolean fixSplitParents = false; // fix lingering split parents
207   private boolean fixReferenceFiles = false; // fix lingering reference store file
208   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
209   private boolean fixTableLocks = false; // fix table locks which are expired
210 
211   // limit checking/fixes to listed tables, if empty attempt to check/fix all
212   // hbase:meta are always checked
213   private Set<TableName> tablesIncluded = new HashSet<TableName>();
214   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
215   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
216   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
217   private Path sidelineDir = null;
218 
219   private boolean rerun = false; // if we tried to fix something, rerun hbck
220   private static boolean summary = false; // if we want to print less output
221   private boolean checkMetaOnly = false;
222   private boolean checkRegionBoundaries = false;
223   private boolean ignorePreCheckPermission = false; // if pre-check permission
224 
225   /*********
226    * State
227    *********/
228   final private ErrorReporter errors;
229   int fixes = 0;
230 
231   /**
232    * This map contains the state of all hbck items.  It maps from encoded region
233    * name to HbckInfo structure.  The information contained in HbckInfo is used
234    * to detect and correct consistency (hdfs/meta/deployment) problems.
235    */
236   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
237   private TreeSet<TableName> disabledTables =
238     new TreeSet<TableName>();
239   // Empty regioninfo qualifiers in hbase:meta
240   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
241 
242   /**
243    * This map from Tablename -> TableInfo contains the structures necessary to
244    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
245    * to prevent dupes.
246    *
247    * If tablesIncluded is empty, this map contains all tables.
248    * Otherwise, it contains only meta tables and tables in tablesIncluded,
249    * unless checkMetaOnly is specified, in which case, it contains only
250    * the meta table
251    */
252   private SortedMap<TableName, TableInfo> tablesInfo =
253       new ConcurrentSkipListMap<TableName, TableInfo>();
254 
255   /**
256    * When initially looking at HDFS, we attempt to find any orphaned data.
257    */
258   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
259 
260   private Map<TableName, Set<String>> orphanTableDirs =
261       new HashMap<TableName, Set<String>>();
262 
263   /**
264    * Constructor
265    *
266    * @param conf Configuration object
267    * @throws MasterNotRunningException if the master is not running
268    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
269    */
270   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
271       ZooKeeperConnectionException, IOException, ClassNotFoundException {
272     super(conf);
273     // make a copy, just to be sure we're not overriding someone else's config
274     setConf(HBaseConfiguration.create(getConf()));
275     // disable blockcache for tool invocation, see HBASE-10500
276     getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
277     errors = getErrorReporter(conf);
278 
279     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
280     executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
281   }
282 
283   /**
284    * Constructor
285    *
286    * @param conf
287    *          Configuration object
288    * @throws MasterNotRunningException
289    *           if the master is not running
290    * @throws ZooKeeperConnectionException
291    *           if unable to connect to ZooKeeper
292    */
293   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
294       ZooKeeperConnectionException, IOException, ClassNotFoundException {
295     super(conf);
296     errors = getErrorReporter(getConf());
297     this.executor = exec;
298   }
299 
300   /**
301    * To repair region consistency, one must call connect() in order to repair
302    * online state.
303    */
304   public void connect() throws IOException {
305     connection = HConnectionManager.createConnection(getConf());
306     admin = new HBaseAdmin(connection);
307     meta = new HTable(TableName.META_TABLE_NAME, connection);
308     status = admin.getClusterStatus();
309   }
310 
311   /**
312    * Get deployed regions according to the region servers.
313    */
314   private void loadDeployedRegions() throws IOException, InterruptedException {
315     // From the master, get a list of all known live region servers
316     Collection<ServerName> regionServers = status.getServers();
317     errors.print("Number of live region servers: " + regionServers.size());
318     if (details) {
319       for (ServerName rsinfo: regionServers) {
320         errors.print("  " + rsinfo.getServerName());
321       }
322     }
323 
324     // From the master, get a list of all dead region servers
325     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
326     errors.print("Number of dead region servers: " + deadRegionServers.size());
327     if (details) {
328       for (ServerName name: deadRegionServers) {
329         errors.print("  " + name);
330       }
331     }
332 
333     // Print the current master name and state
334     errors.print("Master: " + status.getMaster());
335 
336     // Print the list of all backup masters
337     Collection<ServerName> backupMasters = status.getBackupMasters();
338     errors.print("Number of backup masters: " + backupMasters.size());
339     if (details) {
340       for (ServerName name: backupMasters) {
341         errors.print("  " + name);
342       }
343     }
344 
345     errors.print("Average load: " + status.getAverageLoad());
346     errors.print("Number of requests: " + status.getRequestsCount());
347     errors.print("Number of regions: " + status.getRegionsCount());
348 
349     Map<String, RegionState> rits = status.getRegionsInTransition();
350     errors.print("Number of regions in transition: " + rits.size());
351     if (details) {
352       for (RegionState state: rits.values()) {
353         errors.print("  " + state.toDescriptiveString());
354       }
355     }
356 
357     // Determine what's deployed
358     processRegionServers(regionServers);
359   }
360 
361   /**
362    * Clear the current state of hbck.
363    */
364   private void clearState() {
365     // Make sure regionInfo is empty before starting
366     fixes = 0;
367     regionInfoMap.clear();
368     emptyRegionInfoQualifiers.clear();
369     disabledTables.clear();
370     errors.clear();
371     tablesInfo.clear();
372     orphanHdfsDirs.clear();
373   }
374 
375   /**
376    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
377    * the table integrity rules.  HBase doesn't need to be online for this
378    * operation to work.
379    */
380   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
381     // Initial pass to fix orphans.
382     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
383         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
384       LOG.info("Loading regioninfos HDFS");
385       // if nothing is happening this should always complete in two iterations.
386       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
387       int curIter = 0;
388       do {
389         clearState(); // clears hbck state and reset fixes to 0 and.
390         // repair what's on HDFS
391         restoreHdfsIntegrity();
392         curIter++;// limit the number of iterations.
393       } while (fixes > 0 && curIter <= maxIterations);
394 
395       // Repairs should be done in the first iteration and verification in the second.
396       // If there are more than 2 passes, something funny has happened.
397       if (curIter > 2) {
398         if (curIter == maxIterations) {
399           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
400               + "Tables integrity may not be fully repaired!");
401         } else {
402           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
403         }
404       }
405     }
406   }
407 
408   /**
409    * This repair method requires the cluster to be online since it contacts
410    * region servers and the masters.  It makes each region's state in HDFS, in
411    * hbase:meta, and deployments consistent.
412    *
413    * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable
414    * error.  If 0, we have a clean hbase.
415    */
416   public int onlineConsistencyRepair() throws IOException, KeeperException,
417     InterruptedException {
418     clearState();
419 
420     // get regions according to what is online on each RegionServer
421     loadDeployedRegions();
422     // check whether hbase:meta is deployed and online
423     recordMetaRegion();
424     // Check if hbase:meta is found only once and in the right place
425     if (!checkMetaRegion()) {
426       String errorMsg = "hbase:meta table is not consistent. ";
427       if (shouldFixAssignments()) {
428         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
429       } else {
430         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
431       }
432       errors.reportError(errorMsg + " Exiting...");
433       return -2;
434     }
435     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
436     LOG.info("Loading regionsinfo from the hbase:meta table");
437     boolean success = loadMetaEntries();
438     if (!success) return -1;
439 
440     // Empty cells in hbase:meta?
441     reportEmptyMetaCells();
442 
443     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
444     if (shouldFixEmptyMetaCells()) {
445       fixEmptyMetaCells();
446     }
447 
448     // get a list of all tables that have not changed recently.
449     if (!checkMetaOnly) {
450       reportTablesInFlux();
451     }
452 
453     // load regiondirs and regioninfos from HDFS
454     if (shouldCheckHdfs()) {
455       loadHdfsRegionDirs();
456       loadHdfsRegionInfos();
457     }
458 
459     // Get disabled tables from ZooKeeper
460     loadDisabledTables();
461 
462     // fix the orphan tables
463     fixOrphanTables();
464 
465     // Check and fix consistency
466     checkAndFixConsistency();
467 
468     // Check integrity (does not fix)
469     checkIntegrity();
470     return errors.getErrorList().size();
471   }
472 
473   /**
474    * Contacts the master and prints out cluster-wide information
475    * @return 0 on success, non-zero on failure
476    */
477   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
478     // print hbase server version
479     errors.print("Version: " + status.getHBaseVersion());
480     offlineHdfsIntegrityRepair();
481 
482     // turn the balancer off
483     boolean oldBalancer = admin.setBalancerRunning(false, true);
484     try {
485       onlineConsistencyRepair();
486     }
487     finally {
488       admin.setBalancerRunning(oldBalancer, false);
489     }
490 
491     if (checkRegionBoundaries) {
492       checkRegionBoundaries();
493     }
494 
495     offlineReferenceFileRepair();
496 
497     checkAndFixTableLocks();
498 
499     // Print table summary
500     printTableSummary(tablesInfo);
501     return errors.summarize();
502   }
503 
504   public static byte[] keyOnly (byte[] b) {
505     if (b == null)
506       return b;
507     int rowlength = Bytes.toShort(b, 0);
508     byte[] result = new byte[rowlength];
509     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
510     return result;
511   }
512 
513   private static class RegionBoundariesInformation {
514     public byte [] regionName;
515     public byte [] metaFirstKey;
516     public byte [] metaLastKey;
517     public byte [] storesFirstKey;
518     public byte [] storesLastKey;
519     public String toString () {
520       return "regionName=" + Bytes.toStringBinary(regionName) +
521              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
522              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
523              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
524              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
525     }
526   }
527 
528   public void checkRegionBoundaries() {
529     try {
530       ByteArrayComparator comparator = new ByteArrayComparator();
531       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false);
532       final RegionBoundariesInformation currentRegionBoundariesInformation =
533           new RegionBoundariesInformation();
534       Path hbaseRoot = FSUtils.getRootDir(getConf());
535       for (HRegionInfo regionInfo : regions) {
536         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
537         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
538         // For each region, get the start and stop key from the META and compare them to the
539         // same information from the Stores.
540         Path path = new Path(tableDir, regionInfo.getEncodedName());
541         FileSystem fs = path.getFileSystem(getConf());
542         FileStatus[] files = fs.listStatus(path);
543         // For all the column families in this region...
544         byte[] storeFirstKey = null;
545         byte[] storeLastKey = null;
546         for (FileStatus file : files) {
547           String fileName = file.getPath().toString();
548           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
549           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
550             FileStatus[] storeFiles = fs.listStatus(file.getPath());
551             // For all the stores in this column family.
552             for (FileStatus storeFile : storeFiles) {
553               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
554                   getConf()), getConf());
555               if ((reader.getFirstKey() != null)
556                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
557                       reader.getFirstKey()) > 0))) {
558                 storeFirstKey = reader.getFirstKey();
559               }
560               if ((reader.getLastKey() != null)
561                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
562                       reader.getLastKey())) < 0)) {
563                 storeLastKey = reader.getLastKey();
564               }
565               reader.close();
566             }
567           }
568         }
569         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
570         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
571         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
572         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
573         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
574           currentRegionBoundariesInformation.metaFirstKey = null;
575         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
576           currentRegionBoundariesInformation.metaLastKey = null;
577 
578         // For a region to be correct, we need the META start key to be smaller or equal to the
579         // smallest start key from all the stores, and the start key from the next META entry to
580         // be bigger than the last key from all the current stores. First region start key is null;
581         // Last region end key is null; some regions can be empty and not have any store.
582 
583         boolean valid = true;
584         // Checking start key.
585         if ((currentRegionBoundariesInformation.storesFirstKey != null)
586             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
587           valid = valid
588               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
589                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
590         }
591         // Checking stop key.
592         if ((currentRegionBoundariesInformation.storesLastKey != null)
593             && (currentRegionBoundariesInformation.metaLastKey != null)) {
594           valid = valid
595               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
596                 currentRegionBoundariesInformation.metaLastKey) < 0;
597         }
598         if (!valid) {
599           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
600             tablesInfo.get(regionInfo.getTable()));
601           LOG.warn("Region's boundaries not alligned between stores and META for:");
602           LOG.warn(currentRegionBoundariesInformation);
603         }
604       }
605     } catch (IOException e) {
606       LOG.error(e);
607     }
608   }
609 
610   /**
611    * Iterates through the list of all orphan/invalid regiondirs.
612    */
613   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
614     for (HbckInfo hi : orphanHdfsDirs) {
615       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
616       adoptHdfsOrphan(hi);
617     }
618   }
619 
620   /**
621    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
622    * these orphans by creating a new region, and moving the column families,
623    * recovered edits, HLogs, into the new region dir.  We determine the region
624    * startkey and endkeys by looking at all of the hfiles inside the column
625    * families to identify the min and max keys. The resulting region will
626    * likely violate table integrity but will be dealt with by merging
627    * overlapping regions.
628    */
629   @SuppressWarnings("deprecation")
630   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
631     Path p = hi.getHdfsRegionDir();
632     FileSystem fs = p.getFileSystem(getConf());
633     FileStatus[] dirs = fs.listStatus(p);
634     if (dirs == null) {
635       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
636           p + ". This dir could probably be deleted.");
637       return ;
638     }
639 
640     TableName tableName = hi.getTableName();
641     TableInfo tableInfo = tablesInfo.get(tableName);
642     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
643     HTableDescriptor template = tableInfo.getHTD();
644 
645     // find min and max key values
646     Pair<byte[],byte[]> orphanRegionRange = null;
647     for (FileStatus cf : dirs) {
648       String cfName= cf.getPath().getName();
649       // TODO Figure out what the special dirs are
650       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
651 
652       FileStatus[] hfiles = fs.listStatus(cf.getPath());
653       for (FileStatus hfile : hfiles) {
654         byte[] start, end;
655         HFile.Reader hf = null;
656         try {
657           CacheConfig cacheConf = new CacheConfig(getConf());
658           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
659           hf.loadFileInfo();
660           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
661           start = startKv.getRow();
662           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
663           end = endKv.getRow();
664         } catch (IOException ioe) {
665           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
666           continue;
667         } catch (NullPointerException ioe) {
668           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
669           continue;
670         } finally {
671           if (hf != null) {
672             hf.close();
673           }
674         }
675 
676         // expand the range to include the range of all hfiles
677         if (orphanRegionRange == null) {
678           // first range
679           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
680         } else {
681           // TODO add test
682 
683           // expand range only if the hfile is wider.
684           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
685             orphanRegionRange.setFirst(start);
686           }
687           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
688             orphanRegionRange.setSecond(end);
689           }
690         }
691       }
692     }
693     if (orphanRegionRange == null) {
694       LOG.warn("No data in dir " + p + ", sidelining data");
695       fixes++;
696       sidelineRegionDir(fs, hi);
697       return;
698     }
699     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
700         Bytes.toString(orphanRegionRange.getSecond()) + ")");
701 
702     // create new region on hdfs.  move data into place.
703     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
704     LOG.info("Creating new region : " + hri);
705     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
706     Path target = region.getRegionFileSystem().getRegionDir();
707 
708     // rename all the data to new region
709     mergeRegionDirs(target, hi);
710     fixes++;
711   }
712 
713   /**
714    * This method determines if there are table integrity errors in HDFS.  If
715    * there are errors and the appropriate "fix" options are enabled, the method
716    * will first correct orphan regions making them into legit regiondirs, and
717    * then reload to merge potentially overlapping regions.
718    *
719    * @return number of table integrity errors found
720    */
721   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
722     // Determine what's on HDFS
723     LOG.info("Loading HBase regioninfo from HDFS...");
724     loadHdfsRegionDirs(); // populating regioninfo table.
725 
726     int errs = errors.getErrorList().size();
727     // First time just get suggestions.
728     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
729     checkHdfsIntegrity(false, false);
730 
731     if (errors.getErrorList().size() == errs) {
732       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
733       return 0;
734     }
735 
736     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
737       adoptHdfsOrphans(orphanHdfsDirs);
738       // TODO optimize by incrementally adding instead of reloading.
739     }
740 
741     // Make sure there are no holes now.
742     if (shouldFixHdfsHoles()) {
743       clearState(); // this also resets # fixes.
744       loadHdfsRegionDirs();
745       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
746       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
747     }
748 
749     // Now we fix overlaps
750     if (shouldFixHdfsOverlaps()) {
751       // second pass we fix overlaps.
752       clearState(); // this also resets # fixes.
753       loadHdfsRegionDirs();
754       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
755       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
756     }
757 
758     return errors.getErrorList().size();
759   }
760 
761   /**
762    * Scan all the store file names to find any lingering reference files,
763    * which refer to some none-exiting files. If "fix" option is enabled,
764    * any lingering reference file will be sidelined if found.
765    * <p>
766    * Lingering reference file prevents a region from opening. It has to
767    * be fixed before a cluster can start properly.
768    */
769   private void offlineReferenceFileRepair() throws IOException {
770     Configuration conf = getConf();
771     Path hbaseRoot = FSUtils.getRootDir(conf);
772     FileSystem fs = hbaseRoot.getFileSystem(conf);
773     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
774     for (Path path: allFiles.values()) {
775       boolean isReference = false;
776       try {
777         isReference = StoreFileInfo.isReference(path);
778       } catch (Throwable t) {
779         // Ignore. Some files may not be store files at all.
780         // For example, files under .oldlogs folder in hbase:meta
781         // Warning message is already logged by
782         // StoreFile#isReference.
783       }
784       if (!isReference) continue;
785 
786       Path referredToFile = StoreFileInfo.getReferredToFile(path);
787       if (fs.exists(referredToFile)) continue;  // good, expected
788 
789       // Found a lingering reference file
790       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
791         "Found lingering reference file " + path);
792       if (!shouldFixReferenceFiles()) continue;
793 
794       // Now, trying to fix it since requested
795       boolean success = false;
796       String pathStr = path.toString();
797 
798       // A reference file path should be like
799       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
800       // Up 5 directories to get the root folder.
801       // So the file will be sidelined to a similar folder structure.
802       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
803       for (int i = 0; index > 0 && i < 5; i++) {
804         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
805       }
806       if (index > 0) {
807         Path rootDir = getSidelineDir();
808         Path dst = new Path(rootDir, pathStr.substring(index + 1));
809         fs.mkdirs(dst.getParent());
810         LOG.info("Trying to sildeline reference file "
811           + path + " to " + dst);
812         setShouldRerun();
813 
814         success = fs.rename(path, dst);
815       }
816       if (!success) {
817         LOG.error("Failed to sideline reference file " + path);
818       }
819     }
820   }
821 
822   /**
823    * TODO -- need to add tests for this.
824    */
825   private void reportEmptyMetaCells() {
826     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
827       emptyRegionInfoQualifiers.size());
828     if (details) {
829       for (Result r: emptyRegionInfoQualifiers) {
830         errors.print("  " + r);
831       }
832     }
833   }
834 
835   /**
836    * TODO -- need to add tests for this.
837    */
838   private void reportTablesInFlux() {
839     AtomicInteger numSkipped = new AtomicInteger(0);
840     HTableDescriptor[] allTables = getTables(numSkipped);
841     errors.print("Number of Tables: " + allTables.length);
842     if (details) {
843       if (numSkipped.get() > 0) {
844         errors.detail("Number of Tables in flux: " + numSkipped.get());
845       }
846       for (HTableDescriptor td : allTables) {
847         errors.detail("  Table: " + td.getTableName() + "\t" +
848                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
849                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
850                            " families: " + td.getFamilies().size());
851       }
852     }
853   }
854 
855   public ErrorReporter getErrors() {
856     return errors;
857   }
858 
859   /**
860    * Read the .regioninfo file from the file system.  If there is no
861    * .regioninfo, add it to the orphan hdfs region list.
862    */
863   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
864     Path regionDir = hbi.getHdfsRegionDir();
865     if (regionDir == null) {
866       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
867       return;
868     }
869 
870     if (hbi.hdfsEntry.hri != null) {
871       // already loaded data
872       return;
873     }
874 
875     FileSystem fs = FileSystem.get(getConf());
876     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
877     LOG.debug("HRegionInfo read: " + hri.toString());
878     hbi.hdfsEntry.hri = hri;
879   }
880 
881   /**
882    * Exception thrown when a integrity repair operation fails in an
883    * unresolvable way.
884    */
885   public static class RegionRepairException extends IOException {
886     private static final long serialVersionUID = 1L;
887     final IOException ioe;
888     public RegionRepairException(String s, IOException ioe) {
889       super(s);
890       this.ioe = ioe;
891     }
892   }
893 
894   /**
895    * Populate hbi's from regionInfos loaded from file system.
896    */
897   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
898       throws IOException, InterruptedException {
899     tablesInfo.clear(); // regenerating the data
900     // generate region split structure
901     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
902 
903     // Parallelized read of .regioninfo files.
904     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
905     List<Future<Void>> hbiFutures;
906 
907     for (HbckInfo hbi : hbckInfos) {
908       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
909       hbis.add(work);
910     }
911 
912     // Submit and wait for completion
913     hbiFutures = executor.invokeAll(hbis);
914 
915     for(int i=0; i<hbiFutures.size(); i++) {
916       WorkItemHdfsRegionInfo work = hbis.get(i);
917       Future<Void> f = hbiFutures.get(i);
918       try {
919         f.get();
920       } catch(ExecutionException e) {
921         LOG.warn("Failed to read .regioninfo file for region " +
922               work.hbi.getRegionNameAsString(), e.getCause());
923       }
924     }
925 
926     Path hbaseRoot = FSUtils.getRootDir(getConf());
927     FileSystem fs = hbaseRoot.getFileSystem(getConf());
928     // serialized table info gathering.
929     for (HbckInfo hbi: hbckInfos) {
930 
931       if (hbi.getHdfsHRI() == null) {
932         // was an orphan
933         continue;
934       }
935 
936 
937       // get table name from hdfs, populate various HBaseFsck tables.
938       TableName tableName = hbi.getTableName();
939       if (tableName == null) {
940         // There was an entry in hbase:meta not in the HDFS?
941         LOG.warn("tableName was null for: " + hbi);
942         continue;
943       }
944 
945       TableInfo modTInfo = tablesInfo.get(tableName);
946       if (modTInfo == null) {
947         // only executed once per table.
948         modTInfo = new TableInfo(tableName);
949         tablesInfo.put(tableName, modTInfo);
950         try {
951           HTableDescriptor htd =
952               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
953           modTInfo.htds.add(htd);
954         } catch (IOException ioe) {
955           if (!orphanTableDirs.containsKey(tableName)) {
956             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
957             //should only report once for each table
958             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
959                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
960             Set<String> columns = new HashSet<String>();
961             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
962           }
963         }
964       }
965       if (!hbi.isSkipChecks()) {
966         modTInfo.addRegionInfo(hbi);
967       }
968     }
969 
970     loadTableInfosForTablesWithNoRegion();
971 
972     return tablesInfo;
973   }
974 
975   /**
976    * To get the column family list according to the column family dirs
977    * @param columns
978    * @param hbi
979    * @return a set of column families
980    * @throws IOException
981    */
982   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
983     Path regionDir = hbi.getHdfsRegionDir();
984     FileSystem fs = regionDir.getFileSystem(getConf());
985     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
986     for (FileStatus subdir : subDirs) {
987       String columnfamily = subdir.getPath().getName();
988       columns.add(columnfamily);
989     }
990     return columns;
991   }
992 
993   /**
994    * To fabricate a .tableinfo file with following contents<br>
995    * 1. the correct tablename <br>
996    * 2. the correct colfamily list<br>
997    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
998    * @throws IOException
999    */
1000   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1001       Set<String> columns) throws IOException {
1002     if (columns ==null || columns.isEmpty()) return false;
1003     HTableDescriptor htd = new HTableDescriptor(tableName);
1004     for (String columnfamimly : columns) {
1005       htd.addFamily(new HColumnDescriptor(columnfamimly));
1006     }
1007     fstd.createTableDescriptor(htd, true);
1008     return true;
1009   }
1010 
1011   /**
1012    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1013    * @throws IOException
1014    */
1015   public void fixEmptyMetaCells() throws IOException {
1016     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1017       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1018       for (Result region : emptyRegionInfoQualifiers) {
1019         deleteMetaRegion(region.getRow());
1020         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1021       }
1022       emptyRegionInfoQualifiers.clear();
1023     }
1024   }
1025 
1026   /**
1027    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1028    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1029    * 2. else create a default .tableinfo file with following items<br>
1030    * &nbsp;2.1 the correct tablename <br>
1031    * &nbsp;2.2 the correct colfamily list<br>
1032    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1033    * @throws IOException
1034    */
1035   public void fixOrphanTables() throws IOException {
1036     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1037 
1038       List<TableName> tmpList = new ArrayList<TableName>();
1039       tmpList.addAll(orphanTableDirs.keySet());
1040       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1041       Iterator<Entry<TableName, Set<String>>> iter =
1042           orphanTableDirs.entrySet().iterator();
1043       int j = 0;
1044       int numFailedCase = 0;
1045       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1046       while (iter.hasNext()) {
1047         Entry<TableName, Set<String>> entry =
1048             (Entry<TableName, Set<String>>) iter.next();
1049         TableName tableName = entry.getKey();
1050         LOG.info("Trying to fix orphan table error: " + tableName);
1051         if (j < htds.length) {
1052           if (tableName.equals(htds[j].getTableName())) {
1053             HTableDescriptor htd = htds[j];
1054             LOG.info("fixing orphan table: " + tableName + " from cache");
1055             fstd.createTableDescriptor(htd, true);
1056             j++;
1057             iter.remove();
1058           }
1059         } else {
1060           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1061             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1062             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1063             iter.remove();
1064           } else {
1065             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1066             numFailedCase++;
1067           }
1068         }
1069         fixes++;
1070       }
1071 
1072       if (orphanTableDirs.isEmpty()) {
1073         // all orphanTableDirs are luckily recovered
1074         // re-run doFsck after recovering the .tableinfo file
1075         setShouldRerun();
1076         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1077       } else if (numFailedCase > 0) {
1078         LOG.error("Failed to fix " + numFailedCase
1079             + " OrphanTables with default .tableinfo files");
1080       }
1081 
1082     }
1083     //cleanup the list
1084     orphanTableDirs.clear();
1085 
1086   }
1087 
1088   /**
1089    * This borrows code from MasterFileSystem.bootstrap()
1090    *
1091    * @return an open hbase:meta HRegion
1092    */
1093   private HRegion createNewMeta() throws IOException {
1094       Path rootdir = FSUtils.getRootDir(getConf());
1095     Configuration c = getConf();
1096     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1097     MasterFileSystem.setInfoFamilyCachingForMeta(false);
1098     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c,
1099         HTableDescriptor.META_TABLEDESC);
1100     MasterFileSystem.setInfoFamilyCachingForMeta(true);
1101     return meta;
1102   }
1103 
1104   /**
1105    * Generate set of puts to add to new meta.  This expects the tables to be
1106    * clean with no overlaps or holes.  If there are any problems it returns null.
1107    *
1108    * @return An array list of puts to do in bulk, null if tables have problems
1109    */
1110   private ArrayList<Put> generatePuts(
1111       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1112     ArrayList<Put> puts = new ArrayList<Put>();
1113     boolean hasProblems = false;
1114     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1115       TableName name = e.getKey();
1116 
1117       // skip "hbase:meta"
1118       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1119         continue;
1120       }
1121 
1122       TableInfo ti = e.getValue();
1123       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1124           .entrySet()) {
1125         Collection<HbckInfo> his = spl.getValue();
1126         int sz = his.size();
1127         if (sz != 1) {
1128           // problem
1129           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1130               + " had " +  sz + " regions instead of exactly 1." );
1131           hasProblems = true;
1132           continue;
1133         }
1134 
1135         // add the row directly to meta.
1136         HbckInfo hi = his.iterator().next();
1137         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1138         Put p = MetaEditor.makePutFromRegionInfo(hri);
1139         puts.add(p);
1140       }
1141     }
1142     return hasProblems ? null : puts;
1143   }
1144 
1145   /**
1146    * Suggest fixes for each table
1147    */
1148   private void suggestFixes(
1149       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1150     for (TableInfo tInfo : tablesInfo.values()) {
1151       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1152       tInfo.checkRegionChain(handler);
1153     }
1154   }
1155 
1156   /**
1157    * Rebuilds meta from information in hdfs/fs.  Depends on configuration
1158    * settings passed into hbck constructor to point to a particular fs/dir.
1159    *
1160    * @param fix flag that determines if method should attempt to fix holes
1161    * @return true if successful, false if attempt failed.
1162    */
1163   public boolean rebuildMeta(boolean fix) throws IOException,
1164       InterruptedException {
1165 
1166     // TODO check to make sure hbase is offline. (or at least the table
1167     // currently being worked on is off line)
1168 
1169     // Determine what's on HDFS
1170     LOG.info("Loading HBase regioninfo from HDFS...");
1171     loadHdfsRegionDirs(); // populating regioninfo table.
1172 
1173     int errs = errors.getErrorList().size();
1174     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1175     checkHdfsIntegrity(false, false);
1176 
1177     // make sure ok.
1178     if (errors.getErrorList().size() != errs) {
1179       // While in error state, iterate until no more fixes possible
1180       while(true) {
1181         fixes = 0;
1182         suggestFixes(tablesInfo);
1183         errors.clear();
1184         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1185         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1186 
1187         int errCount = errors.getErrorList().size();
1188 
1189         if (fixes == 0) {
1190           if (errCount > 0) {
1191             return false; // failed to fix problems.
1192           } else {
1193             break; // no fixes and no problems? drop out and fix stuff!
1194           }
1195         }
1196       }
1197     }
1198 
1199     // we can rebuild, move old meta out of the way and start
1200     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1201     Path backupDir = sidelineOldMeta();
1202 
1203     LOG.info("Creating new hbase:meta");
1204     HRegion meta = createNewMeta();
1205 
1206     // populate meta
1207     List<Put> puts = generatePuts(tablesInfo);
1208     if (puts == null) {
1209       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1210         "You may need to restore the previously sidelined hbase:meta");
1211       return false;
1212     }
1213     meta.batchMutate(puts.toArray(new Put[0]));
1214     HRegion.closeHRegion(meta);
1215     LOG.info("Success! hbase:meta table rebuilt.");
1216     LOG.info("Old hbase:meta is moved into " + backupDir);
1217     return true;
1218   }
1219 
1220   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1221       boolean fixOverlaps) throws IOException {
1222     LOG.info("Checking HBase region split map from HDFS data...");
1223     for (TableInfo tInfo : tablesInfo.values()) {
1224       TableIntegrityErrorHandler handler;
1225       if (fixHoles || fixOverlaps) {
1226         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1227           fixHoles, fixOverlaps);
1228       } else {
1229         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1230       }
1231       if (!tInfo.checkRegionChain(handler)) {
1232         // should dump info as well.
1233         errors.report("Found inconsistency in table " + tInfo.getName());
1234       }
1235     }
1236     return tablesInfo;
1237   }
1238 
1239   private Path getSidelineDir() throws IOException {
1240     if (sidelineDir == null) {
1241       Path hbaseDir = FSUtils.getRootDir(getConf());
1242       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1243       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1244           + startMillis);
1245     }
1246     return sidelineDir;
1247   }
1248 
1249   /**
1250    * Sideline a region dir (instead of deleting it)
1251    */
1252   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1253     return sidelineRegionDir(fs, null, hi);
1254   }
1255 
1256   /**
1257    * Sideline a region dir (instead of deleting it)
1258    *
1259    * @param parentDir if specified, the region will be sidelined to
1260    * folder like .../parentDir/<table name>/<region name>. The purpose
1261    * is to group together similar regions sidelined, for example, those
1262    * regions should be bulk loaded back later on. If null, it is ignored.
1263    */
1264   Path sidelineRegionDir(FileSystem fs,
1265       String parentDir, HbckInfo hi) throws IOException {
1266     TableName tableName = hi.getTableName();
1267     Path regionDir = hi.getHdfsRegionDir();
1268 
1269     if (!fs.exists(regionDir)) {
1270       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1271       return null;
1272     }
1273 
1274     Path rootDir = getSidelineDir();
1275     if (parentDir != null) {
1276       rootDir = new Path(rootDir, parentDir);
1277     }
1278     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1279     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1280     fs.mkdirs(sidelineRegionDir);
1281     boolean success = false;
1282     FileStatus[] cfs =  fs.listStatus(regionDir);
1283     if (cfs == null) {
1284       LOG.info("Region dir is empty: " + regionDir);
1285     } else {
1286       for (FileStatus cf : cfs) {
1287         Path src = cf.getPath();
1288         Path dst =  new Path(sidelineRegionDir, src.getName());
1289         if (fs.isFile(src)) {
1290           // simple file
1291           success = fs.rename(src, dst);
1292           if (!success) {
1293             String msg = "Unable to rename file " + src +  " to " + dst;
1294             LOG.error(msg);
1295             throw new IOException(msg);
1296           }
1297           continue;
1298         }
1299 
1300         // is a directory.
1301         fs.mkdirs(dst);
1302 
1303         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1304         // FileSystem.rename is inconsistent with directories -- if the
1305         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1306         // it moves the src into the dst dir resulting in (foo/a/b).  If
1307         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1308         FileStatus[] hfiles = fs.listStatus(src);
1309         if (hfiles != null && hfiles.length > 0) {
1310           for (FileStatus hfile : hfiles) {
1311             success = fs.rename(hfile.getPath(), dst);
1312             if (!success) {
1313               String msg = "Unable to rename file " + src +  " to " + dst;
1314               LOG.error(msg);
1315               throw new IOException(msg);
1316             }
1317           }
1318         }
1319         LOG.debug("Sideline directory contents:");
1320         debugLsr(sidelineRegionDir);
1321       }
1322     }
1323 
1324     LOG.info("Removing old region dir: " + regionDir);
1325     success = fs.delete(regionDir, true);
1326     if (!success) {
1327       String msg = "Unable to delete dir " + regionDir;
1328       LOG.error(msg);
1329       throw new IOException(msg);
1330     }
1331     return sidelineRegionDir;
1332   }
1333 
1334   /**
1335    * Side line an entire table.
1336    */
1337   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1338       Path backupHbaseDir) throws IOException {
1339     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1340     if (fs.exists(tableDir)) {
1341       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1342       fs.mkdirs(backupTableDir.getParent());
1343       boolean success = fs.rename(tableDir, backupTableDir);
1344       if (!success) {
1345         throw new IOException("Failed to move  " + tableName + " from "
1346             +  tableDir + " to " + backupTableDir);
1347       }
1348     } else {
1349       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1350     }
1351   }
1352 
1353   /**
1354    * @return Path to backup of original directory
1355    */
1356   Path sidelineOldMeta() throws IOException {
1357     // put current hbase:meta aside.
1358     Path hbaseDir = FSUtils.getRootDir(getConf());
1359     FileSystem fs = hbaseDir.getFileSystem(getConf());
1360     Path backupDir = getSidelineDir();
1361     fs.mkdirs(backupDir);
1362 
1363     try {
1364       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1365     } catch (IOException e) {
1366         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1367             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1368             + hbaseDir.getName() + ".", e);
1369       throw e; // throw original exception
1370     }
1371     return backupDir;
1372   }
1373 
1374   /**
1375    * Load the list of disabled tables in ZK into local set.
1376    * @throws ZooKeeperConnectionException
1377    * @throws IOException
1378    */
1379   private void loadDisabledTables()
1380   throws ZooKeeperConnectionException, IOException {
1381     HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1382       @Override
1383       public Void connect(HConnection connection) throws IOException {
1384         ZooKeeperWatcher zkw = createZooKeeperWatcher();
1385         try {
1386           for (TableName tableName :
1387               ZKTableReadOnly.getDisabledOrDisablingTables(zkw)) {
1388             disabledTables.add(tableName);
1389           }
1390         } catch (KeeperException ke) {
1391           throw new IOException(ke);
1392         } catch (InterruptedException e) {
1393           throw new InterruptedIOException();
1394         } finally {
1395           zkw.close();
1396         }
1397         return null;
1398       }
1399     });
1400   }
1401 
1402   /**
1403    * Check if the specified region's table is disabled.
1404    */
1405   private boolean isTableDisabled(HRegionInfo regionInfo) {
1406     return disabledTables.contains(regionInfo.getTable());
1407   }
1408 
1409   /**
1410    * Scan HDFS for all regions, recording their information into
1411    * regionInfoMap
1412    */
1413   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1414     Path rootDir = FSUtils.getRootDir(getConf());
1415     FileSystem fs = rootDir.getFileSystem(getConf());
1416 
1417     // list all tables from HDFS
1418     List<FileStatus> tableDirs = Lists.newArrayList();
1419 
1420     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1421 
1422     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1423     for (Path path : paths) {
1424       TableName tableName = FSUtils.getTableName(path);
1425        if ((!checkMetaOnly &&
1426            isTableIncluded(tableName)) ||
1427            tableName.equals(TableName.META_TABLE_NAME)) {
1428          tableDirs.add(fs.getFileStatus(path));
1429        }
1430     }
1431 
1432     // verify that version file exists
1433     if (!foundVersionFile) {
1434       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1435           "Version file does not exist in root dir " + rootDir);
1436       if (shouldFixVersionFile()) {
1437         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1438             + " file.");
1439         setShouldRerun();
1440         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1441             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1442             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1443             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1444       }
1445     }
1446 
1447     // level 1:  <HBASE_DIR>/*
1448     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1449     List<Future<Void>> dirsFutures;
1450 
1451     for (FileStatus tableDir : tableDirs) {
1452       LOG.debug("Loading region dirs from " +tableDir.getPath());
1453       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1454     }
1455 
1456     // Invoke and wait for Callables to complete
1457     dirsFutures = executor.invokeAll(dirs);
1458 
1459     for(Future<Void> f: dirsFutures) {
1460       try {
1461         f.get();
1462       } catch(ExecutionException e) {
1463         LOG.warn("Could not load region dir " , e.getCause());
1464       }
1465     }
1466   }
1467 
1468   /**
1469    * Record the location of the hbase:meta region as found in ZooKeeper.
1470    */
1471   private boolean recordMetaRegion() throws IOException {
1472     HRegionLocation metaLocation = connection.locateRegion(
1473       TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW);
1474 
1475     // Check if Meta region is valid and existing
1476     if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1477         metaLocation.getHostname() == null) {
1478       errors.reportError(ERROR_CODE.NULL_META_REGION,
1479         "META region or some of its attributes are null.");
1480       return false;
1481     }
1482     ServerName sn;
1483     try {
1484       sn = getMetaRegionServerName();
1485     } catch (KeeperException e) {
1486       throw new IOException(e);
1487     }
1488     MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis());
1489     HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1490     if (hbckInfo == null) {
1491       regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1492     } else {
1493       hbckInfo.metaEntry = m;
1494     }
1495     return true;
1496   }
1497 
1498   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1499     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1500       @Override
1501       public void abort(String why, Throwable e) {
1502         LOG.error(why, e);
1503         System.exit(1);
1504       }
1505 
1506       @Override
1507       public boolean isAborted() {
1508         return false;
1509       }
1510 
1511     });
1512   }
1513 
1514   private ServerName getMetaRegionServerName()
1515   throws IOException, KeeperException {
1516     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1517     ServerName sn = null;
1518     try {
1519       sn = MetaRegionTracker.getMetaRegionLocation(zkw);
1520     } finally {
1521       zkw.close();
1522     }
1523     return sn;
1524   }
1525 
1526   /**
1527    * Contacts each regionserver and fetches metadata about regions.
1528    * @param regionServerList - the list of region servers to connect to
1529    * @throws IOException if a remote or network exception occurs
1530    */
1531   void processRegionServers(Collection<ServerName> regionServerList)
1532     throws IOException, InterruptedException {
1533 
1534     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1535     List<Future<Void>> workFutures;
1536 
1537     // loop to contact each region server in parallel
1538     for (ServerName rsinfo: regionServerList) {
1539       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1540     }
1541 
1542     workFutures = executor.invokeAll(workItems);
1543 
1544     for(int i=0; i<workFutures.size(); i++) {
1545       WorkItemRegion item = workItems.get(i);
1546       Future<Void> f = workFutures.get(i);
1547       try {
1548         f.get();
1549       } catch(ExecutionException e) {
1550         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1551             e.getCause());
1552       }
1553     }
1554   }
1555 
1556   /**
1557    * Check consistency of all regions that have been found in previous phases.
1558    */
1559   private void checkAndFixConsistency()
1560   throws IOException, KeeperException, InterruptedException {
1561     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1562       checkRegionConsistency(e.getKey(), e.getValue());
1563     }
1564   }
1565 
1566   private void preCheckPermission() throws IOException, AccessControlException {
1567     if (shouldIgnorePreCheckPermission()) {
1568       return;
1569     }
1570 
1571     Path hbaseDir = FSUtils.getRootDir(getConf());
1572     FileSystem fs = hbaseDir.getFileSystem(getConf());
1573     UserProvider userProvider = UserProvider.instantiate(getConf());
1574     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1575     FileStatus[] files = fs.listStatus(hbaseDir);
1576     for (FileStatus file : files) {
1577       try {
1578         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1579       } catch (AccessControlException ace) {
1580         LOG.warn("Got AccessControlException when preCheckPermission ", ace);
1581         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1582           + " does not have write perms to " + file.getPath()
1583           + ". Please rerun hbck as hdfs user " + file.getOwner());
1584         throw new AccessControlException(ace);
1585       }
1586     }
1587   }
1588 
1589   /**
1590    * Deletes region from meta table
1591    */
1592   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1593     deleteMetaRegion(hi.metaEntry.getRegionName());
1594   }
1595 
1596   /**
1597    * Deletes region from meta table
1598    */
1599   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1600     Delete d = new Delete(metaKey);
1601     meta.delete(d);
1602     meta.flushCommits();
1603     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1604   }
1605 
1606   /**
1607    * Reset the split parent region info in meta table
1608    */
1609   private void resetSplitParent(HbckInfo hi) throws IOException {
1610     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1611     Delete d = new Delete(hi.metaEntry.getRegionName());
1612     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1613     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1614     mutations.add(d);
1615 
1616     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1617     hri.setOffline(false);
1618     hri.setSplit(false);
1619     Put p = MetaEditor.makePutFromRegionInfo(hri);
1620     mutations.add(p);
1621 
1622     meta.mutateRow(mutations);
1623     meta.flushCommits();
1624     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1625   }
1626 
1627   /**
1628    * This backwards-compatibility wrapper for permanently offlining a region
1629    * that should not be alive.  If the region server does not support the
1630    * "offline" method, it will use the closest unassign method instead.  This
1631    * will basically work until one attempts to disable or delete the affected
1632    * table.  The problem has to do with in-memory only master state, so
1633    * restarting the HMaster or failing over to another should fix this.
1634    */
1635   private void offline(byte[] regionName) throws IOException {
1636     String regionString = Bytes.toStringBinary(regionName);
1637     if (!rsSupportsOffline) {
1638       LOG.warn("Using unassign region " + regionString
1639           + " instead of using offline method, you should"
1640           + " restart HMaster after these repairs");
1641       admin.unassign(regionName, true);
1642       return;
1643     }
1644 
1645     // first time we assume the rs's supports #offline.
1646     try {
1647       LOG.info("Offlining region " + regionString);
1648       admin.offline(regionName);
1649     } catch (IOException ioe) {
1650       String notFoundMsg = "java.lang.NoSuchMethodException: " +
1651         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
1652       if (ioe.getMessage().contains(notFoundMsg)) {
1653         LOG.warn("Using unassign region " + regionString
1654             + " instead of using offline method, you should"
1655             + " restart HMaster after these repairs");
1656         rsSupportsOffline = false; // in the future just use unassign
1657         admin.unassign(regionName, true);
1658         return;
1659       }
1660       throw ioe;
1661     }
1662   }
1663 
1664   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
1665     for (OnlineEntry rse : hi.deployedEntries) {
1666       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
1667       try {
1668         HBaseFsckRepair.closeRegionSilentlyAndWait(admin, rse.hsa, rse.hri);
1669         offline(rse.hri.getRegionName());
1670       } catch (IOException ioe) {
1671         LOG.warn("Got exception when attempting to offline region "
1672             + Bytes.toString(rse.hri.getRegionName()), ioe);
1673       }
1674     }
1675   }
1676 
1677   /**
1678    * Attempts to undeploy a region from a region server based in information in
1679    * META.  Any operations that modify the file system should make sure that
1680    * its corresponding region is not deployed to prevent data races.
1681    *
1682    * A separate call is required to update the master in-memory region state
1683    * kept in the AssignementManager.  Because disable uses this state instead of
1684    * that found in META, we can't seem to cleanly disable/delete tables that
1685    * have been hbck fixed.  When used on a version of HBase that does not have
1686    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
1687    * restart or failover may be required.
1688    */
1689   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
1690     if (hi.metaEntry == null && hi.hdfsEntry == null) {
1691       undeployRegions(hi);
1692       return;
1693     }
1694 
1695     // get assignment info and hregioninfo from meta.
1696     Get get = new Get(hi.getRegionName());
1697     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1698     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
1699     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
1700     Result r = meta.get(get);
1701     ServerName serverName = HRegionInfo.getServerName(r);
1702     if (serverName == null) {
1703       errors.reportError("Unable to close region "
1704           + hi.getRegionNameAsString() +  " because meta does not "
1705           + "have handle to reach it.");
1706       return;
1707     }
1708 
1709     HRegionInfo hri = HRegionInfo.getHRegionInfo(r);
1710     if (hri == null) {
1711       LOG.warn("Unable to close region " + hi.getRegionNameAsString()
1712           + " because hbase:meta had invalid or missing "
1713           + HConstants.CATALOG_FAMILY_STR + ":"
1714           + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
1715           + " qualifier value.");
1716       return;
1717     }
1718 
1719     // close the region -- close files and remove assignment
1720     HBaseFsckRepair.closeRegionSilentlyAndWait(admin, serverName, hri);
1721   }
1722 
1723   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
1724     KeeperException, InterruptedException {
1725     // If we are trying to fix the errors
1726     if (shouldFixAssignments()) {
1727       errors.print(msg);
1728       undeployRegions(hbi);
1729       setShouldRerun();
1730       HRegionInfo hri = hbi.getHdfsHRI();
1731       if (hri == null) {
1732         hri = hbi.metaEntry;
1733       }
1734       HBaseFsckRepair.fixUnassigned(admin, hri);
1735       HBaseFsckRepair.waitUntilAssigned(admin, hri);
1736     }
1737   }
1738 
1739   /**
1740    * Check a single region for consistency and correct deployment.
1741    */
1742   private void checkRegionConsistency(final String key, final HbckInfo hbi)
1743   throws IOException, KeeperException, InterruptedException {
1744     String descriptiveName = hbi.toString();
1745 
1746     boolean inMeta = hbi.metaEntry != null;
1747     // In case not checking HDFS, assume the region is on HDFS
1748     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
1749     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
1750     boolean isDeployed = !hbi.deployedOn.isEmpty();
1751     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
1752     boolean deploymentMatchesMeta =
1753       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
1754       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
1755     boolean splitParent =
1756       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
1757     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
1758     boolean recentlyModified = inHdfs &&
1759       hbi.getModTime() + timelag > System.currentTimeMillis();
1760 
1761     // ========== First the healthy cases =============
1762     if (hbi.containsOnlyHdfsEdits()) {
1763       return;
1764     }
1765     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
1766       return;
1767     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
1768       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
1769         "tabled that is not deployed");
1770       return;
1771     } else if (recentlyModified) {
1772       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
1773       return;
1774     }
1775     // ========== Cases where the region is not in hbase:meta =============
1776     else if (!inMeta && !inHdfs && !isDeployed) {
1777       // We shouldn't have record of this region at all then!
1778       assert false : "Entry for region with no data";
1779     } else if (!inMeta && !inHdfs && isDeployed) {
1780       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
1781           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
1782           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1783       if (shouldFixAssignments()) {
1784         undeployRegions(hbi);
1785       }
1786 
1787     } else if (!inMeta && inHdfs && !isDeployed) {
1788       if (hbi.isMerged()) {
1789         // This region has already been merged, the remaining hdfs file will be
1790         // cleaned by CatalogJanitor later
1791         hbi.setSkipChecks(true);
1792         LOG.info("Region " + descriptiveName
1793             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
1794         return;
1795       }
1796       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
1797           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
1798           "or deployed on any region server");
1799       // restore region consistency of an adopted orphan
1800       if (shouldFixMeta()) {
1801         if (!hbi.isHdfsRegioninfoPresent()) {
1802           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
1803               +  " in table integrity repair phase if -fixHdfsOrphans was" +
1804               " used.");
1805           return;
1806         }
1807 
1808         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
1809         HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI());
1810 
1811         tryAssignmentRepair(hbi, "Trying to reassign region...");
1812       }
1813 
1814     } else if (!inMeta && inHdfs && isDeployed) {
1815       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
1816           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1817       debugLsr(hbi.getHdfsRegionDir());
1818       if (shouldFixMeta()) {
1819         if (!hbi.isHdfsRegioninfoPresent()) {
1820           LOG.error("This should have been repaired in table integrity repair phase");
1821           return;
1822         }
1823 
1824         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
1825         HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI());
1826 
1827         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1828       }
1829 
1830     // ========== Cases where the region is in hbase:meta =============
1831     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
1832       // check whether this is an actual error, or just transient state where parent
1833       // is not cleaned
1834       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
1835         // check that split daughters are there
1836         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
1837         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
1838         if (infoA != null && infoB != null) {
1839           // we already processed or will process daughters. Move on, nothing to see here.
1840           hbi.setSkipChecks(true);
1841           return;
1842         }
1843       }
1844       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
1845           + descriptiveName + " is a split parent in META, in HDFS, "
1846           + "and not deployed on any region server. This could be transient.");
1847       if (shouldFixSplitParents()) {
1848         setShouldRerun();
1849         resetSplitParent(hbi);
1850       }
1851     } else if (inMeta && !inHdfs && !isDeployed) {
1852       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
1853           + descriptiveName + " found in META, but not in HDFS "
1854           + "or deployed on any region server.");
1855       if (shouldFixMeta()) {
1856         deleteMetaRegion(hbi);
1857       }
1858     } else if (inMeta && !inHdfs && isDeployed) {
1859       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
1860           + " found in META, but not in HDFS, " +
1861           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1862       // We treat HDFS as ground truth.  Any information in meta is transient
1863       // and equivalent data can be regenerated.  So, lets unassign and remove
1864       // these problems from META.
1865       if (shouldFixAssignments()) {
1866         errors.print("Trying to fix unassigned region...");
1867         closeRegion(hbi);// Close region will cause RS to abort.
1868       }
1869       if (shouldFixMeta()) {
1870         // wait for it to complete
1871         deleteMetaRegion(hbi);
1872       }
1873     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
1874       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
1875           + " not deployed on any region server.");
1876       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1877     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
1878       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
1879           "Region " + descriptiveName + " should not be deployed according " +
1880           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1881       if (shouldFixAssignments()) {
1882         errors.print("Trying to close the region " + descriptiveName);
1883         setShouldRerun();
1884         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1885       }
1886     } else if (inMeta && inHdfs && isMultiplyDeployed) {
1887       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
1888           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
1889           + " but is multiply assigned to region servers " +
1890           Joiner.on(", ").join(hbi.deployedOn));
1891       // If we are trying to fix the errors
1892       if (shouldFixAssignments()) {
1893         errors.print("Trying to fix assignment error...");
1894         setShouldRerun();
1895         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1896       }
1897     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
1898       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
1899           + descriptiveName + " listed in hbase:meta on region server " +
1900           hbi.metaEntry.regionServer + " but found on region server " +
1901           hbi.deployedOn.get(0));
1902       // If we are trying to fix the errors
1903       if (shouldFixAssignments()) {
1904         errors.print("Trying to fix assignment error...");
1905         setShouldRerun();
1906         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1907         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
1908       }
1909     } else {
1910       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
1911           " is in an unforeseen state:" +
1912           " inMeta=" + inMeta +
1913           " inHdfs=" + inHdfs +
1914           " isDeployed=" + isDeployed +
1915           " isMultiplyDeployed=" + isMultiplyDeployed +
1916           " deploymentMatchesMeta=" + deploymentMatchesMeta +
1917           " shouldBeDeployed=" + shouldBeDeployed);
1918     }
1919   }
1920 
1921   /**
1922    * Checks tables integrity. Goes over all regions and scans the tables.
1923    * Collects all the pieces for each table and checks if there are missing,
1924    * repeated or overlapping ones.
1925    * @throws IOException
1926    */
1927   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
1928     tablesInfo = new TreeMap<TableName,TableInfo> ();
1929     List<HbckInfo> noHDFSRegionInfos = new ArrayList<HbckInfo>();
1930     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
1931     for (HbckInfo hbi : regionInfoMap.values()) {
1932       // Check only valid, working regions
1933       if (hbi.metaEntry == null) {
1934         // this assumes that consistency check has run loadMetaEntry
1935         noHDFSRegionInfos.add(hbi);
1936         Path p = hbi.getHdfsRegionDir();
1937         if (p == null) {
1938           errors.report("No regioninfo in Meta or HDFS. " + hbi);
1939         }
1940 
1941         // TODO test.
1942         continue;
1943       }
1944       if (hbi.metaEntry.regionServer == null) {
1945         errors.detail("Skipping region because no region server: " + hbi);
1946         continue;
1947       }
1948       if (hbi.metaEntry.isOffline()) {
1949         errors.detail("Skipping region because it is offline: " + hbi);
1950         continue;
1951       }
1952       if (hbi.containsOnlyHdfsEdits()) {
1953         errors.detail("Skipping region because it only contains edits" + hbi);
1954         continue;
1955       }
1956 
1957       // Missing regionDir or over-deployment is checked elsewhere. Include
1958       // these cases in modTInfo, so we can evaluate those regions as part of
1959       // the region chain in META
1960       //if (hbi.foundRegionDir == null) continue;
1961       //if (hbi.deployedOn.size() != 1) continue;
1962       if (hbi.deployedOn.size() == 0) continue;
1963 
1964       // We should be safe here
1965       TableName tableName = hbi.metaEntry.getTable();
1966       TableInfo modTInfo = tablesInfo.get(tableName);
1967       if (modTInfo == null) {
1968         modTInfo = new TableInfo(tableName);
1969       }
1970       for (ServerName server : hbi.deployedOn) {
1971         modTInfo.addServer(server);
1972       }
1973 
1974       if (!hbi.isSkipChecks()) {
1975         modTInfo.addRegionInfo(hbi);
1976       }
1977 
1978       tablesInfo.put(tableName, modTInfo);
1979     }
1980 
1981     loadTableInfosForTablesWithNoRegion();
1982 
1983     for (TableInfo tInfo : tablesInfo.values()) {
1984       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1985       if (!tInfo.checkRegionChain(handler)) {
1986         errors.report("Found inconsistency in table " + tInfo.getName());
1987       }
1988     }
1989     return tablesInfo;
1990   }
1991 
1992   /** Loads table info's for tables that may not have been included, since there are no
1993    * regions reported for the table, but table dir is there in hdfs
1994    */
1995   private void loadTableInfosForTablesWithNoRegion() throws IOException {
1996     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
1997     for (HTableDescriptor htd : allTables.values()) {
1998       if (checkMetaOnly && !htd.isMetaTable()) {
1999         continue;
2000       }
2001 
2002       TableName tableName = htd.getTableName();
2003       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2004         TableInfo tableInfo = new TableInfo(tableName);
2005         tableInfo.htds.add(htd);
2006         tablesInfo.put(htd.getTableName(), tableInfo);
2007       }
2008     }
2009   }
2010 
2011   /**
2012    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2013    * @return number of file move fixes done to merge regions.
2014    */
2015   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2016     int fileMoves = 0;
2017     String thread = Thread.currentThread().getName();
2018     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2019     debugLsr(contained.getHdfsRegionDir());
2020 
2021     // rename the contained into the container.
2022     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2023     FileStatus[] dirs = null;
2024     try { 
2025       dirs = fs.listStatus(contained.getHdfsRegionDir());
2026     } catch (FileNotFoundException fnfe) {
2027       // region we are attempting to merge in is not present!  Since this is a merge, there is
2028       // no harm skipping this region if it does not exist.
2029       if (!fs.exists(contained.getHdfsRegionDir())) {
2030         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir() 
2031             + " is missing. Assuming already sidelined or moved.");
2032       } else {
2033         sidelineRegionDir(fs, contained);
2034       }
2035       return fileMoves;
2036     }
2037 
2038     if (dirs == null) {
2039       if (!fs.exists(contained.getHdfsRegionDir())) {
2040         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir() 
2041             + " already sidelined.");
2042       } else {
2043         sidelineRegionDir(fs, contained);
2044       }
2045       return fileMoves;
2046     }
2047 
2048     for (FileStatus cf : dirs) {
2049       Path src = cf.getPath();
2050       Path dst =  new Path(targetRegionDir, src.getName());
2051 
2052       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2053         // do not copy the old .regioninfo file.
2054         continue;
2055       }
2056 
2057       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2058         // do not copy the .oldlogs files
2059         continue;
2060       }
2061 
2062       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2063       // FileSystem.rename is inconsistent with directories -- if the
2064       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2065       // it moves the src into the dst dir resulting in (foo/a/b).  If
2066       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2067       for (FileStatus hfile : fs.listStatus(src)) {
2068         boolean success = fs.rename(hfile.getPath(), dst);
2069         if (success) {
2070           fileMoves++;
2071         }
2072       }
2073       LOG.debug("[" + thread + "] Sideline directory contents:");
2074       debugLsr(targetRegionDir);
2075     }
2076 
2077     // if all success.
2078     sidelineRegionDir(fs, contained);
2079     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2080         getSidelineDir());
2081     debugLsr(contained.getHdfsRegionDir());
2082 
2083     return fileMoves;
2084   }
2085 
2086 
2087   static class WorkItemOverlapMerge implements Callable<Void> {
2088     private TableIntegrityErrorHandler handler;
2089     Collection<HbckInfo> overlapgroup;
2090     
2091     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2092       this.handler = handler;
2093       this.overlapgroup = overlapgroup;
2094     }
2095     
2096     @Override
2097     public Void call() throws Exception {
2098       handler.handleOverlapGroup(overlapgroup);
2099       return null;
2100     }
2101   };
2102   
2103   
2104   /**
2105    * Maintain information about a particular table.
2106    */
2107   public class TableInfo {
2108     TableName tableName;
2109     TreeSet <ServerName> deployedOn;
2110 
2111     // backwards regions
2112     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2113 
2114     // sidelined big overlapped regions
2115     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2116 
2117     // region split calculator
2118     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2119 
2120     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2121     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2122 
2123     // key = start split, values = set of splits in problem group
2124     final Multimap<byte[], HbckInfo> overlapGroups =
2125       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2126 
2127     TableInfo(TableName name) {
2128       this.tableName = name;
2129       deployedOn = new TreeSet <ServerName>();
2130     }
2131 
2132     /**
2133      * @return descriptor common to all regions.  null if are none or multiple!
2134      */
2135     private HTableDescriptor getHTD() {
2136       if (htds.size() == 1) {
2137         return (HTableDescriptor)htds.toArray()[0];
2138       } else {
2139         LOG.error("None/Multiple table descriptors found for table '"
2140           + tableName + "' regions: " + htds);
2141       }
2142       return null;
2143     }
2144 
2145     public void addRegionInfo(HbckInfo hir) {
2146       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2147         // end key is absolute end key, just add it.
2148         sc.add(hir);
2149         return;
2150       }
2151 
2152       // if not the absolute end key, check for cycle
2153       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2154         errors.reportError(
2155             ERROR_CODE.REGION_CYCLE,
2156             String.format("The endkey for this region comes before the "
2157                 + "startkey, startkey=%s, endkey=%s",
2158                 Bytes.toStringBinary(hir.getStartKey()),
2159                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2160         backwards.add(hir);
2161         return;
2162       }
2163 
2164       // main case, add to split calculator
2165       sc.add(hir);
2166     }
2167 
2168     public void addServer(ServerName server) {
2169       this.deployedOn.add(server);
2170     }
2171 
2172     public TableName getName() {
2173       return tableName;
2174     }
2175 
2176     public int getNumRegions() {
2177       return sc.getStarts().size() + backwards.size();
2178     }
2179 
2180     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2181       ErrorReporter errors;
2182 
2183       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2184         this.errors = errors;
2185         setTableInfo(ti);
2186       }
2187 
2188       @Override
2189       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2190         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2191             "First region should start with an empty key.  You need to "
2192             + " create a new region and regioninfo in HDFS to plug the hole.",
2193             getTableInfo(), hi);
2194       }
2195 
2196       @Override
2197       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2198         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2199             "Last region should end with an empty key. You need to "
2200                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2201       }
2202 
2203       @Override
2204       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2205         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2206             "Region has the same start and end key.", getTableInfo(), hi);
2207       }
2208 
2209       @Override
2210       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2211         byte[] key = r1.getStartKey();
2212         // dup start key
2213         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2214             "Multiple regions have the same startkey: "
2215             + Bytes.toStringBinary(key), getTableInfo(), r1);
2216         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2217             "Multiple regions have the same startkey: "
2218             + Bytes.toStringBinary(key), getTableInfo(), r2);
2219       }
2220 
2221       @Override
2222       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2223         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2224             "There is an overlap in the region chain.",
2225             getTableInfo(), hi1, hi2);
2226       }
2227 
2228       @Override
2229       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2230         errors.reportError(
2231             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2232             "There is a hole in the region chain between "
2233                 + Bytes.toStringBinary(holeStart) + " and "
2234                 + Bytes.toStringBinary(holeStop)
2235                 + ".  You need to create a new .regioninfo and region "
2236                 + "dir in hdfs to plug the hole.");
2237       }
2238     };
2239 
2240     /**
2241      * This handler fixes integrity errors from hdfs information.  There are
2242      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2243      * 3) invalid regions.
2244      *
2245      * This class overrides methods that fix holes and the overlap group case.
2246      * Individual cases of particular overlaps are handled by the general
2247      * overlap group merge repair case.
2248      *
2249      * If hbase is online, this forces regions offline before doing merge
2250      * operations.
2251      */
2252     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2253       Configuration conf;
2254 
2255       boolean fixOverlaps = true;
2256 
2257       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2258           boolean fixHoles, boolean fixOverlaps) {
2259         super(ti, errors);
2260         this.conf = conf;
2261         this.fixOverlaps = fixOverlaps;
2262         // TODO properly use fixHoles
2263       }
2264 
2265       /**
2266        * This is a special case hole -- when the first region of a table is
2267        * missing from META, HBase doesn't acknowledge the existance of the
2268        * table.
2269        */
2270       @Override
2271       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2272         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2273             "First region should start with an empty key.  Creating a new " +
2274             "region and regioninfo in HDFS to plug the hole.",
2275             getTableInfo(), next);
2276         HTableDescriptor htd = getTableInfo().getHTD();
2277         // from special EMPTY_START_ROW to next region's startKey
2278         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2279             HConstants.EMPTY_START_ROW, next.getStartKey());
2280 
2281         // TODO test
2282         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2283         LOG.info("Table region start key was not empty.  Created new empty region: "
2284             + newRegion + " " +region);
2285         fixes++;
2286       }
2287 
2288       @Override
2289       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2290         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2291             "Last region should end with an empty key.  Creating a new "
2292                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2293         HTableDescriptor htd = getTableInfo().getHTD();
2294         // from curEndKey to EMPTY_START_ROW
2295         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2296             HConstants.EMPTY_START_ROW);
2297 
2298         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2299         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2300             + " " + region);
2301         fixes++;
2302       }
2303 
2304       /**
2305        * There is a hole in the hdfs regions that violates the table integrity
2306        * rules.  Create a new empty region that patches the hole.
2307        */
2308       @Override
2309       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2310         errors.reportError(
2311             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2312             "There is a hole in the region chain between "
2313                 + Bytes.toStringBinary(holeStartKey) + " and "
2314                 + Bytes.toStringBinary(holeStopKey)
2315                 + ".  Creating a new regioninfo and region "
2316                 + "dir in hdfs to plug the hole.");
2317         HTableDescriptor htd = getTableInfo().getHTD();
2318         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2319         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2320         LOG.info("Plugged hold by creating new empty region: "+ newRegion + " " +region);
2321         fixes++;
2322       }
2323 
2324       /**
2325        * This takes set of overlapping regions and merges them into a single
2326        * region.  This covers cases like degenerate regions, shared start key,
2327        * general overlaps, duplicate ranges, and partial overlapping regions.
2328        *
2329        * Cases:
2330        * - Clean regions that overlap
2331        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2332        * 
2333        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2334        */
2335       @Override
2336       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2337           throws IOException {
2338         Preconditions.checkNotNull(overlap);
2339         Preconditions.checkArgument(overlap.size() >0);
2340 
2341         if (!this.fixOverlaps) {
2342           LOG.warn("Not attempting to repair overlaps.");
2343           return;
2344         }
2345 
2346         if (overlap.size() > maxMerge) {
2347           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2348             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2349           if (sidelineBigOverlaps) {
2350             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2351             sidelineBigOverlaps(overlap);
2352           }
2353           return;
2354         }
2355 
2356         mergeOverlaps(overlap);
2357       }
2358 
2359       void mergeOverlaps(Collection<HbckInfo> overlap)
2360           throws IOException {
2361         String thread = Thread.currentThread().getName();
2362         LOG.info("== [" + thread + "] Merging regions into one region: "
2363           + Joiner.on(",").join(overlap));
2364         // get the min / max range and close all concerned regions
2365         Pair<byte[], byte[]> range = null;
2366         for (HbckInfo hi : overlap) {
2367           if (range == null) {
2368             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2369           } else {
2370             if (RegionSplitCalculator.BYTES_COMPARATOR
2371                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2372               range.setFirst(hi.getStartKey());
2373             }
2374             if (RegionSplitCalculator.BYTES_COMPARATOR
2375                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2376               range.setSecond(hi.getEndKey());
2377             }
2378           }
2379           // need to close files so delete can happen.
2380           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2381           LOG.debug("[" + thread + "] Contained region dir before close");
2382           debugLsr(hi.getHdfsRegionDir());
2383           try {
2384             LOG.info("[" + thread + "] Closing region: " + hi);
2385             closeRegion(hi);
2386           } catch (IOException ioe) {
2387             LOG.warn("[" + thread + "] Was unable to close region " + hi
2388               + ".  Just continuing... ", ioe);
2389           } catch (InterruptedException e) {
2390             LOG.warn("[" + thread + "] Was unable to close region " + hi
2391               + ".  Just continuing... ", e);
2392           }
2393 
2394           try {
2395             LOG.info("[" + thread + "] Offlining region: " + hi);
2396             offline(hi.getRegionName());
2397           } catch (IOException ioe) {
2398             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2399               + ".  Just continuing... ", ioe);
2400           }
2401         }
2402 
2403         // create new empty container region.
2404         HTableDescriptor htd = getTableInfo().getHTD();
2405         // from start key to end Key
2406         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2407             range.getSecond());
2408         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2409         LOG.info("[" + thread + "] Created new empty container region: " +
2410             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2411         debugLsr(region.getRegionFileSystem().getRegionDir());
2412 
2413         // all target regions are closed, should be able to safely cleanup.
2414         boolean didFix= false;
2415         Path target = region.getRegionFileSystem().getRegionDir();
2416         for (HbckInfo contained : overlap) {
2417           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2418           int merges = mergeRegionDirs(target, contained);
2419           if (merges > 0) {
2420             didFix = true;
2421           }
2422         }
2423         if (didFix) {
2424           fixes++;
2425         }
2426       }
2427 
2428       /**
2429        * Sideline some regions in a big overlap group so that it
2430        * will have fewer regions, and it is easier to merge them later on.
2431        *
2432        * @param bigOverlap the overlapped group with regions more than maxMerge
2433        * @throws IOException
2434        */
2435       void sidelineBigOverlaps(
2436           Collection<HbckInfo> bigOverlap) throws IOException {
2437         int overlapsToSideline = bigOverlap.size() - maxMerge;
2438         if (overlapsToSideline > maxOverlapsToSideline) {
2439           overlapsToSideline = maxOverlapsToSideline;
2440         }
2441         List<HbckInfo> regionsToSideline =
2442           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2443         FileSystem fs = FileSystem.get(conf);
2444         for (HbckInfo regionToSideline: regionsToSideline) {
2445           try {
2446             LOG.info("Closing region: " + regionToSideline);
2447             closeRegion(regionToSideline);
2448           } catch (IOException ioe) {
2449             LOG.warn("Was unable to close region " + regionToSideline
2450               + ".  Just continuing... ", ioe);
2451           } catch (InterruptedException e) {
2452             LOG.warn("Was unable to close region " + regionToSideline
2453               + ".  Just continuing... ", e);
2454           }
2455 
2456           try {
2457             LOG.info("Offlining region: " + regionToSideline);
2458             offline(regionToSideline.getRegionName());
2459           } catch (IOException ioe) {
2460             LOG.warn("Unable to offline region from master: " + regionToSideline
2461               + ".  Just continuing... ", ioe);
2462           }
2463 
2464           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2465           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2466           if (sidelineRegionDir != null) {
2467             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2468             LOG.info("After sidelined big overlapped region: "
2469               + regionToSideline.getRegionNameAsString()
2470               + " to " + sidelineRegionDir.toString());
2471             fixes++;
2472           }
2473         }
2474       }
2475     }
2476 
2477     /**
2478      * Check the region chain (from META) of this table.  We are looking for
2479      * holes, overlaps, and cycles.
2480      * @return false if there are errors
2481      * @throws IOException
2482      */
2483     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2484       // When table is disabled no need to check for the region chain. Some of the regions
2485       // accidently if deployed, this below code might report some issues like missing start
2486       // or end regions or region hole in chain and may try to fix which is unwanted.
2487       if (disabledTables.contains(this.tableName)) {
2488         return true;
2489       }
2490       int originalErrorsCount = errors.getErrorList().size();
2491       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2492       SortedSet<byte[]> splits = sc.getSplits();
2493 
2494       byte[] prevKey = null;
2495       byte[] problemKey = null;
2496 
2497       if (splits.size() == 0) {
2498         // no region for this table
2499         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2500       }
2501 
2502       for (byte[] key : splits) {
2503         Collection<HbckInfo> ranges = regions.get(key);
2504         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2505           for (HbckInfo rng : ranges) {
2506             handler.handleRegionStartKeyNotEmpty(rng);
2507           }
2508         }
2509 
2510         // check for degenerate ranges
2511         for (HbckInfo rng : ranges) {
2512           // special endkey case converts '' to null
2513           byte[] endKey = rng.getEndKey();
2514           endKey = (endKey.length == 0) ? null : endKey;
2515           if (Bytes.equals(rng.getStartKey(),endKey)) {
2516             handler.handleDegenerateRegion(rng);
2517           }
2518         }
2519 
2520         if (ranges.size() == 1) {
2521           // this split key is ok -- no overlap, not a hole.
2522           if (problemKey != null) {
2523             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2524           }
2525           problemKey = null; // fell through, no more problem.
2526         } else if (ranges.size() > 1) {
2527           // set the new problem key group name, if already have problem key, just
2528           // keep using it.
2529           if (problemKey == null) {
2530             // only for overlap regions.
2531             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2532             problemKey = key;
2533           }
2534           overlapGroups.putAll(problemKey, ranges);
2535 
2536           // record errors
2537           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
2538           //  this dumb and n^2 but this shouldn't happen often
2539           for (HbckInfo r1 : ranges) {
2540             subRange.remove(r1);
2541             for (HbckInfo r2 : subRange) {
2542               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
2543                 handler.handleDuplicateStartKeys(r1,r2);
2544               } else {
2545                 // overlap
2546                 handler.handleOverlapInRegionChain(r1, r2);
2547               }
2548             }
2549           }
2550 
2551         } else if (ranges.size() == 0) {
2552           if (problemKey != null) {
2553             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2554           }
2555           problemKey = null;
2556 
2557           byte[] holeStopKey = sc.getSplits().higher(key);
2558           // if higher key is null we reached the top.
2559           if (holeStopKey != null) {
2560             // hole
2561             handler.handleHoleInRegionChain(key, holeStopKey);
2562           }
2563         }
2564         prevKey = key;
2565       }
2566 
2567       // When the last region of a table is proper and having an empty end key, 'prevKey'
2568       // will be null.
2569       if (prevKey != null) {
2570         handler.handleRegionEndKeyNotEmpty(prevKey);
2571       }
2572 
2573       // TODO fold this into the TableIntegrityHandler
2574       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
2575         LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
2576             " false to run serially.");
2577         boolean ok = handleOverlapsParallel(handler, prevKey);
2578         if (!ok) {
2579           return false;
2580         }
2581       } else {
2582         LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
2583             " true to run in parallel.");
2584         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2585           handler.handleOverlapGroup(overlap);
2586         }
2587       }
2588 
2589       if (details) {
2590         // do full region split map dump
2591         errors.print("---- Table '"  +  this.tableName
2592             + "': region split map");
2593         dump(splits, regions);
2594         errors.print("---- Table '"  +  this.tableName
2595             + "': overlap groups");
2596         dumpOverlapProblems(overlapGroups);
2597         errors.print("There are " + overlapGroups.keySet().size()
2598             + " overlap groups with " + overlapGroups.size()
2599             + " overlapping regions");
2600       }
2601       if (!sidelinedRegions.isEmpty()) {
2602         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
2603         errors.print("---- Table '"  +  this.tableName
2604             + "': sidelined big overlapped regions");
2605         dumpSidelinedRegions(sidelinedRegions);
2606       }
2607       return errors.getErrorList().size() == originalErrorsCount;
2608     }
2609 
2610     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
2611         throws IOException {
2612       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
2613       // safely assume each group is independent. 
2614       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
2615       List<Future<Void>> rets;
2616       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2617         // 
2618         merges.add(new WorkItemOverlapMerge(overlap, handler));
2619       }
2620       try {
2621         rets = executor.invokeAll(merges);
2622       } catch (InterruptedException e) {
2623         LOG.error("Overlap merges were interrupted", e);
2624         return false;
2625       }
2626       for(int i=0; i<merges.size(); i++) {
2627         WorkItemOverlapMerge work = merges.get(i);
2628         Future<Void> f = rets.get(i);
2629         try {
2630           f.get();
2631         } catch(ExecutionException e) {
2632           LOG.warn("Failed to merge overlap group" + work, e.getCause());
2633         } catch (InterruptedException e) {
2634           LOG.error("Waiting for overlap merges was interrupted", e);
2635           return false;
2636         }
2637       }
2638       return true;
2639     }
2640 
2641     /**
2642      * This dumps data in a visually reasonable way for visual debugging
2643      *
2644      * @param splits
2645      * @param regions
2646      */
2647     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
2648       // we display this way because the last end key should be displayed as well.
2649       StringBuilder sb = new StringBuilder();
2650       for (byte[] k : splits) {
2651         sb.setLength(0); // clear out existing buffer, if any.
2652         sb.append(Bytes.toStringBinary(k) + ":\t");
2653         for (HbckInfo r : regions.get(k)) {
2654           sb.append("[ "+ r.toString() + ", "
2655               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
2656         }
2657         errors.print(sb.toString());
2658       }
2659     }
2660   }
2661 
2662   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
2663     // we display this way because the last end key should be displayed as
2664     // well.
2665     for (byte[] k : regions.keySet()) {
2666       errors.print(Bytes.toStringBinary(k) + ":");
2667       for (HbckInfo r : regions.get(k)) {
2668         errors.print("[ " + r.toString() + ", "
2669             + Bytes.toStringBinary(r.getEndKey()) + "]");
2670       }
2671       errors.print("----");
2672     }
2673   }
2674 
2675   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
2676     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
2677       TableName tableName = entry.getValue().getTableName();
2678       Path path = entry.getKey();
2679       errors.print("This sidelined region dir should be bulk loaded: "
2680         + path.toString());
2681       errors.print("Bulk load command looks like: "
2682         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
2683         + path.toUri().getPath() + " "+ tableName);
2684     }
2685   }
2686 
2687   public Multimap<byte[], HbckInfo> getOverlapGroups(
2688       TableName table) {
2689     TableInfo ti = tablesInfo.get(table);
2690     return ti.overlapGroups;
2691   }
2692 
2693   /**
2694    * Return a list of user-space table names whose metadata have not been
2695    * modified in the last few milliseconds specified by timelag
2696    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2697    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2698    * milliseconds specified by timelag, then the table is a candidate to be returned.
2699    * @return tables that have not been modified recently
2700    * @throws IOException if an error is encountered
2701    */
2702   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
2703     List<TableName> tableNames = new ArrayList<TableName>();
2704     long now = System.currentTimeMillis();
2705 
2706     for (HbckInfo hbi : regionInfoMap.values()) {
2707       MetaEntry info = hbi.metaEntry;
2708 
2709       // if the start key is zero, then we have found the first region of a table.
2710       // pick only those tables that were not modified in the last few milliseconds.
2711       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
2712         if (info.modTime + timelag < now) {
2713           tableNames.add(info.getTable());
2714         } else {
2715           numSkipped.incrementAndGet(); // one more in-flux table
2716         }
2717       }
2718     }
2719     return getHTableDescriptors(tableNames);
2720   }
2721 
2722   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
2723     HTableDescriptor[] htd = new HTableDescriptor[0];
2724     try {
2725       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
2726       htd = new HBaseAdmin(getConf()).getTableDescriptorsByTableName(tableNames);
2727     } catch (IOException e) {
2728       LOG.debug("Exception getting table descriptors", e);
2729     }
2730     return htd;
2731   }
2732 
2733   /**
2734    * Gets the entry in regionInfo corresponding to the the given encoded
2735    * region name. If the region has not been seen yet, a new entry is added
2736    * and returned.
2737    */
2738   private synchronized HbckInfo getOrCreateInfo(String name) {
2739     HbckInfo hbi = regionInfoMap.get(name);
2740     if (hbi == null) {
2741       hbi = new HbckInfo(null);
2742       regionInfoMap.put(name, hbi);
2743     }
2744     return hbi;
2745   }
2746 
2747   private void checkAndFixTableLocks() throws IOException {
2748     TableLockChecker checker = new TableLockChecker(createZooKeeperWatcher(), errors);
2749     checker.checkTableLocks();
2750 
2751     if (this.fixTableLocks) {
2752       checker.fixExpiredTableLocks();
2753     }
2754   }
2755 
2756   /**
2757     * Check values in regionInfo for hbase:meta
2758     * Check if zero or more than one regions with hbase:meta are found.
2759     * If there are inconsistencies (i.e. zero or more than one regions
2760     * pretend to be holding the hbase:meta) try to fix that and report an error.
2761     * @throws IOException from HBaseFsckRepair functions
2762    * @throws KeeperException
2763    * @throws InterruptedException
2764     */
2765   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
2766     List<HbckInfo> metaRegions = Lists.newArrayList();
2767     for (HbckInfo value : regionInfoMap.values()) {
2768       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
2769         metaRegions.add(value);
2770       }
2771     }
2772 
2773     // There will be always one entry in regionInfoMap corresponding to hbase:meta
2774     // Check the deployed servers. It should be exactly one server.
2775     List<ServerName> servers = new ArrayList<ServerName>();
2776     HbckInfo metaHbckInfo = null;
2777     if (!metaRegions.isEmpty()) {
2778       metaHbckInfo = metaRegions.get(0);
2779       servers = metaHbckInfo.deployedOn;
2780     }
2781     if (servers.size() != 1) {
2782       if (servers.size() == 0) {
2783         errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta is not found on any region.");
2784         if (shouldFixAssignments()) {
2785           errors.print("Trying to fix a problem with hbase:meta..");
2786           setShouldRerun();
2787           // try to fix it (treat it as unassigned region)
2788           HBaseFsckRepair.fixUnassigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2789           HBaseFsckRepair.waitUntilAssigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2790         }
2791       } else if (servers.size() > 1) {
2792         errors
2793             .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta is found on more than one region.");
2794         if (shouldFixAssignments()) {
2795           if (metaHbckInfo == null) {
2796             errors.print(
2797               "Unable to fix problem with hbase:meta due to hbase:meta region info missing");
2798             return false;
2799           }
2800           errors.print("Trying to fix a problem with hbase:meta..");
2801           setShouldRerun();
2802           // try fix it (treat is a dupe assignment)
2803           HBaseFsckRepair.fixMultiAssignment(admin, metaHbckInfo.metaEntry, servers);
2804         }
2805       }
2806       // rerun hbck with hopefully fixed META
2807       return false;
2808     }
2809     // no errors, so continue normally
2810     return true;
2811   }
2812 
2813   /**
2814    * Scan hbase:meta, adding all regions found to the regionInfo map.
2815    * @throws IOException if an error is encountered
2816    */
2817   boolean loadMetaEntries() throws IOException {
2818     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2819       int countRecord = 1;
2820 
2821       // comparator to sort KeyValues with latest modtime
2822       final Comparator<Cell> comp = new Comparator<Cell>() {
2823         @Override
2824         public int compare(Cell k1, Cell k2) {
2825           return (int)(k1.getTimestamp() - k2.getTimestamp());
2826         }
2827       };
2828 
2829       @Override
2830       public boolean processRow(Result result) throws IOException {
2831         try {
2832 
2833           // record the latest modification of this META record
2834           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
2835           Pair<HRegionInfo, ServerName> pair = HRegionInfo.getHRegionInfoAndServerName(result);
2836           if (pair == null || pair.getFirst() == null) {
2837             emptyRegionInfoQualifiers.add(result);
2838             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2839               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2840             return true;
2841           }
2842           ServerName sn = null;
2843           if (pair.getSecond() != null) {
2844             sn = pair.getSecond();
2845           }
2846           HRegionInfo hri = pair.getFirst();
2847           if (!(isTableIncluded(hri.getTable())
2848               || hri.isMetaRegion())) {
2849             return true;
2850           }
2851           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
2852           MetaEntry m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
2853           HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
2854           if (previous == null) {
2855             regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
2856           } else if (previous.metaEntry == null) {
2857             previous.metaEntry = m;
2858           } else {
2859             throw new IOException("Two entries in hbase:meta are same " + previous);
2860           }
2861           
2862           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
2863           for (HRegionInfo mergeRegion : new HRegionInfo[] {
2864               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
2865             if (mergeRegion != null) {
2866               // This region is already been merged
2867               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
2868               hbInfo.setMerged(true);
2869             }
2870           }
2871 
2872           // show proof of progress to the user, once for every 100 records.
2873           if (countRecord % 100 == 0) {
2874             errors.progress();
2875           }
2876           countRecord++;
2877           return true;
2878         } catch (RuntimeException e) {
2879           LOG.error("Result=" + result);
2880           throw e;
2881         }
2882       }
2883     };
2884     if (!checkMetaOnly) {
2885       // Scan hbase:meta to pick up user regions
2886       MetaScanner.metaScan(getConf(), visitor);
2887     }
2888 
2889     errors.print("");
2890     return true;
2891   }
2892 
2893   /**
2894    * Stores the regioninfo entries scanned from META
2895    */
2896   static class MetaEntry extends HRegionInfo {
2897     ServerName regionServer;   // server hosting this region
2898     long modTime;          // timestamp of most recent modification metadata
2899     HRegionInfo splitA, splitB; //split daughters
2900 
2901     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
2902       this(rinfo, regionServer, modTime, null, null);
2903     }
2904 
2905     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
2906         HRegionInfo splitA, HRegionInfo splitB) {
2907       super(rinfo);
2908       this.regionServer = regionServer;
2909       this.modTime = modTime;
2910       this.splitA = splitA;
2911       this.splitB = splitB;
2912     }
2913 
2914     @Override
2915     public boolean equals(Object o) {
2916       boolean superEq = super.equals(o);
2917       if (!superEq) {
2918         return superEq;
2919       }
2920 
2921       MetaEntry me = (MetaEntry) o;
2922       if (!regionServer.equals(me.regionServer)) {
2923         return false;
2924       }
2925       return (modTime == me.modTime);
2926     }
2927 
2928     @Override
2929     public int hashCode() {
2930       int hash = Arrays.hashCode(getRegionName());
2931       hash ^= getRegionId();
2932       hash ^= Arrays.hashCode(getStartKey());
2933       hash ^= Arrays.hashCode(getEndKey());
2934       hash ^= Boolean.valueOf(isOffline()).hashCode();
2935       hash ^= getTable().hashCode();
2936       if (regionServer != null) {
2937         hash ^= regionServer.hashCode();
2938       }
2939       hash ^= modTime;
2940       return hash;
2941     }
2942   }
2943 
2944   /**
2945    * Stores the regioninfo entries from HDFS
2946    */
2947   static class HdfsEntry {
2948     HRegionInfo hri;
2949     Path hdfsRegionDir = null;
2950     long hdfsRegionDirModTime  = 0;
2951     boolean hdfsRegioninfoFilePresent = false;
2952     boolean hdfsOnlyEdits = false;
2953   }
2954 
2955   /**
2956    * Stores the regioninfo retrieved from Online region servers.
2957    */
2958   static class OnlineEntry {
2959     HRegionInfo hri;
2960     ServerName hsa;
2961 
2962     @Override
2963     public String toString() {
2964       return hsa.toString() + ";" + hri.getRegionNameAsString();
2965     }
2966   }
2967 
2968   /**
2969    * Maintain information about a particular region.  It gathers information
2970    * from three places -- HDFS, META, and region servers.
2971    */
2972   public static class HbckInfo implements KeyRange {
2973     private MetaEntry metaEntry = null; // info in META
2974     private HdfsEntry hdfsEntry = null; // info in HDFS
2975     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
2976     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
2977     private boolean skipChecks = false; // whether to skip further checks to this region info.
2978     private boolean isMerged = false;// whether this region has already been merged into another one
2979 
2980     HbckInfo(MetaEntry metaEntry) {
2981       this.metaEntry = metaEntry;
2982     }
2983 
2984     public synchronized void addServer(HRegionInfo hri, ServerName server) {
2985       OnlineEntry rse = new OnlineEntry() ;
2986       rse.hri = hri;
2987       rse.hsa = server;
2988       this.deployedEntries.add(rse);
2989       this.deployedOn.add(server);
2990     }
2991 
2992     @Override
2993     public synchronized String toString() {
2994       StringBuilder sb = new StringBuilder();
2995       sb.append("{ meta => ");
2996       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
2997       sb.append( ", hdfs => " + getHdfsRegionDir());
2998       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
2999       sb.append(" }");
3000       return sb.toString();
3001     }
3002 
3003     @Override
3004     public byte[] getStartKey() {
3005       if (this.metaEntry != null) {
3006         return this.metaEntry.getStartKey();
3007       } else if (this.hdfsEntry != null) {
3008         return this.hdfsEntry.hri.getStartKey();
3009       } else {
3010         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3011         return null;
3012       }
3013     }
3014 
3015     @Override
3016     public byte[] getEndKey() {
3017       if (this.metaEntry != null) {
3018         return this.metaEntry.getEndKey();
3019       } else if (this.hdfsEntry != null) {
3020         return this.hdfsEntry.hri.getEndKey();
3021       } else {
3022         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3023         return null;
3024       }
3025     }
3026 
3027     public TableName getTableName() {
3028       if (this.metaEntry != null) {
3029         return this.metaEntry.getTable();
3030       } else if (this.hdfsEntry != null) {
3031         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3032         // so we get the name from the Path
3033         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3034         return FSUtils.getTableName(tableDir);
3035       } else {
3036         // Currently no code exercises this path, but we could add one for
3037         // getting table name from OnlineEntry
3038         return null;
3039       }
3040     }
3041 
3042     public String getRegionNameAsString() {
3043       if (metaEntry != null) {
3044         return metaEntry.getRegionNameAsString();
3045       } else if (hdfsEntry != null) {
3046         if (hdfsEntry.hri != null) {
3047           return hdfsEntry.hri.getRegionNameAsString();
3048         }
3049       }
3050       return null;
3051     }
3052 
3053     public byte[] getRegionName() {
3054       if (metaEntry != null) {
3055         return metaEntry.getRegionName();
3056       } else if (hdfsEntry != null) {
3057         return hdfsEntry.hri.getRegionName();
3058       } else {
3059         return null;
3060       }
3061     }
3062 
3063     Path getHdfsRegionDir() {
3064       if (hdfsEntry == null) {
3065         return null;
3066       }
3067       return hdfsEntry.hdfsRegionDir;
3068     }
3069 
3070     boolean containsOnlyHdfsEdits() {
3071       if (hdfsEntry == null) {
3072         return false;
3073       }
3074       return hdfsEntry.hdfsOnlyEdits;
3075     }
3076 
3077     boolean isHdfsRegioninfoPresent() {
3078       if (hdfsEntry == null) {
3079         return false;
3080       }
3081       return hdfsEntry.hdfsRegioninfoFilePresent;
3082     }
3083 
3084     long getModTime() {
3085       if (hdfsEntry == null) {
3086         return 0;
3087       }
3088       return hdfsEntry.hdfsRegionDirModTime;
3089     }
3090 
3091     HRegionInfo getHdfsHRI() {
3092       if (hdfsEntry == null) {
3093         return null;
3094       }
3095       return hdfsEntry.hri;
3096     }
3097 
3098     public void setSkipChecks(boolean skipChecks) {
3099       this.skipChecks = skipChecks;
3100     }
3101 
3102     public boolean isSkipChecks() {
3103       return skipChecks;
3104     }
3105 
3106     public void setMerged(boolean isMerged) {
3107       this.isMerged = isMerged;
3108     }
3109 
3110     public boolean isMerged() {
3111       return this.isMerged;
3112     }
3113   }
3114 
3115   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3116     @Override
3117     public int compare(HbckInfo l, HbckInfo r) {
3118       if (l == r) {
3119         // same instance
3120         return 0;
3121       }
3122 
3123       int tableCompare = l.getTableName().compareTo(r.getTableName());
3124       if (tableCompare != 0) {
3125         return tableCompare;
3126       }
3127 
3128       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3129           l.getStartKey(), r.getStartKey());
3130       if (startComparison != 0) {
3131         return startComparison;
3132       }
3133 
3134       // Special case for absolute endkey
3135       byte[] endKey = r.getEndKey();
3136       endKey = (endKey.length == 0) ? null : endKey;
3137       byte[] endKey2 = l.getEndKey();
3138       endKey2 = (endKey2.length == 0) ? null : endKey2;
3139       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3140           endKey2,  endKey);
3141 
3142       if (endComparison != 0) {
3143         return endComparison;
3144       }
3145 
3146       // use regionId as tiebreaker.
3147       // Null is considered after all possible values so make it bigger.
3148       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3149         return 0;
3150       }
3151       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3152         return 1;
3153       }
3154       // l.hdfsEntry must not be null
3155       if (r.hdfsEntry == null) {
3156         return -1;
3157       }
3158       // both l.hdfsEntry and r.hdfsEntry must not be null.
3159       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3160     }
3161   };
3162 
3163   /**
3164    * Prints summary of all tables found on the system.
3165    */
3166   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3167     StringBuilder sb = new StringBuilder();
3168     errors.print("Summary:");
3169     for (TableInfo tInfo : tablesInfo.values()) {
3170       if (errors.tableHasErrors(tInfo)) {
3171         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3172       } else {
3173         errors.print("  " + tInfo.getName() + " is okay.");
3174       }
3175       errors.print("    Number of regions: " + tInfo.getNumRegions());
3176       sb.setLength(0); // clear out existing buffer, if any.
3177       sb.append("    Deployed on: ");
3178       for (ServerName server : tInfo.deployedOn) {
3179         sb.append(" " + server.toString());
3180       }
3181       errors.print(sb.toString());
3182     }
3183   }
3184 
3185   static ErrorReporter getErrorReporter(
3186       final Configuration conf) throws ClassNotFoundException {
3187     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3188     return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf);
3189   }
3190 
3191   public interface ErrorReporter {
3192     enum ERROR_CODE {
3193       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3194       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3195       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3196       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3197       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3198       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3199       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR
3200     }
3201     void clear();
3202     void report(String message);
3203     void reportError(String message);
3204     void reportError(ERROR_CODE errorCode, String message);
3205     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3206     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3207     void reportError(
3208       ERROR_CODE errorCode,
3209       String message,
3210       TableInfo table,
3211       HbckInfo info1,
3212       HbckInfo info2
3213     );
3214     int summarize();
3215     void detail(String details);
3216     ArrayList<ERROR_CODE> getErrorList();
3217     void progress();
3218     void print(String message);
3219     void resetErrors();
3220     boolean tableHasErrors(TableInfo table);
3221   }
3222 
3223   static class PrintingErrorReporter implements ErrorReporter {
3224     public int errorCount = 0;
3225     private int showProgress;
3226 
3227     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3228 
3229     // for use by unit tests to verify which errors were discovered
3230     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3231 
3232     @Override
3233     public void clear() {
3234       errorTables.clear();
3235       errorList.clear();
3236       errorCount = 0;
3237     }
3238 
3239     @Override
3240     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3241       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3242         System.err.println(message);
3243         return;
3244       }
3245 
3246       errorList.add(errorCode);
3247       if (!summary) {
3248         System.out.println("ERROR: " + message);
3249       }
3250       errorCount++;
3251       showProgress = 0;
3252     }
3253 
3254     @Override
3255     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3256       errorTables.add(table);
3257       reportError(errorCode, message);
3258     }
3259 
3260     @Override
3261     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3262                                          HbckInfo info) {
3263       errorTables.add(table);
3264       String reference = "(region " + info.getRegionNameAsString() + ")";
3265       reportError(errorCode, reference + " " + message);
3266     }
3267 
3268     @Override
3269     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3270                                          HbckInfo info1, HbckInfo info2) {
3271       errorTables.add(table);
3272       String reference = "(regions " + info1.getRegionNameAsString()
3273           + " and " + info2.getRegionNameAsString() + ")";
3274       reportError(errorCode, reference + " " + message);
3275     }
3276 
3277     @Override
3278     public synchronized void reportError(String message) {
3279       reportError(ERROR_CODE.UNKNOWN, message);
3280     }
3281 
3282     /**
3283      * Report error information, but do not increment the error count.  Intended for cases
3284      * where the actual error would have been reported previously.
3285      * @param message
3286      */
3287     @Override
3288     public synchronized void report(String message) {
3289       if (! summary) {
3290         System.out.println("ERROR: " + message);
3291       }
3292       showProgress = 0;
3293     }
3294 
3295     @Override
3296     public synchronized int summarize() {
3297       System.out.println(Integer.toString(errorCount) +
3298                          " inconsistencies detected.");
3299       if (errorCount == 0) {
3300         System.out.println("Status: OK");
3301         return 0;
3302       } else {
3303         System.out.println("Status: INCONSISTENT");
3304         return -1;
3305       }
3306     }
3307 
3308     @Override
3309     public ArrayList<ERROR_CODE> getErrorList() {
3310       return errorList;
3311     }
3312 
3313     @Override
3314     public synchronized void print(String message) {
3315       if (!summary) {
3316         System.out.println(message);
3317       }
3318     }
3319 
3320     @Override
3321     public boolean tableHasErrors(TableInfo table) {
3322       return errorTables.contains(table);
3323     }
3324 
3325     @Override
3326     public void resetErrors() {
3327       errorCount = 0;
3328     }
3329 
3330     @Override
3331     public synchronized void detail(String message) {
3332       if (details) {
3333         System.out.println(message);
3334       }
3335       showProgress = 0;
3336     }
3337 
3338     @Override
3339     public synchronized void progress() {
3340       if (showProgress++ == 10) {
3341         if (!summary) {
3342           System.out.print(".");
3343         }
3344         showProgress = 0;
3345       }
3346     }
3347   }
3348 
3349   /**
3350    * Contact a region server and get all information from it
3351    */
3352   static class WorkItemRegion implements Callable<Void> {
3353     private HBaseFsck hbck;
3354     private ServerName rsinfo;
3355     private ErrorReporter errors;
3356     private HConnection connection;
3357 
3358     WorkItemRegion(HBaseFsck hbck, ServerName info,
3359                    ErrorReporter errors, HConnection connection) {
3360       this.hbck = hbck;
3361       this.rsinfo = info;
3362       this.errors = errors;
3363       this.connection = connection;
3364     }
3365 
3366     @Override
3367     public synchronized Void call() throws IOException {
3368       errors.progress();
3369       try {
3370         BlockingInterface server = connection.getAdmin(rsinfo);
3371 
3372         // list all online regions from this region server
3373         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3374         regions = filterRegions(regions);
3375 
3376         if (details) {
3377           errors.detail("RegionServer: " + rsinfo.getServerName() +
3378                            " number of regions: " + regions.size());
3379           for (HRegionInfo rinfo: regions) {
3380             errors.detail("  " + rinfo.getRegionNameAsString() +
3381                              " id: " + rinfo.getRegionId() +
3382                              " encoded_name: " + rinfo.getEncodedName() +
3383                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3384                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3385           }
3386         }
3387 
3388         // check to see if the existence of this region matches the region in META
3389         for (HRegionInfo r:regions) {
3390           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3391           hbi.addServer(r, rsinfo);
3392         }
3393       } catch (IOException e) {          // unable to connect to the region server.
3394         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3395           " Unable to fetch region information. " + e);
3396         throw e;
3397       }
3398       return null;
3399     }
3400 
3401     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
3402       List<HRegionInfo> ret = Lists.newArrayList();
3403       for (HRegionInfo hri : regions) {
3404         if (hri.isMetaTable() || (!hbck.checkMetaOnly
3405             && hbck.isTableIncluded(hri.getTable()))) {
3406           ret.add(hri);
3407         }
3408       }
3409       return ret;
3410     }
3411   }
3412 
3413   /**
3414    * Contact hdfs and get all information about specified table directory into
3415    * regioninfo list.
3416    */
3417   static class WorkItemHdfsDir implements Callable<Void> {
3418     private HBaseFsck hbck;
3419     private FileStatus tableDir;
3420     private ErrorReporter errors;
3421     private FileSystem fs;
3422 
3423     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
3424                     FileStatus status) {
3425       this.hbck = hbck;
3426       this.fs = fs;
3427       this.tableDir = status;
3428       this.errors = errors;
3429     }
3430 
3431     @Override
3432     public synchronized Void call() throws IOException {
3433       try {
3434         // level 2: <HBASE_DIR>/<table>/*
3435         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3436         for (FileStatus regionDir : regionDirs) {
3437           String encodedName = regionDir.getPath().getName();
3438           // ignore directories that aren't hexadecimal
3439           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
3440             continue;
3441           }
3442 
3443           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3444           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
3445           HdfsEntry he = new HdfsEntry();
3446           synchronized (hbi) {
3447             if (hbi.getHdfsRegionDir() != null) {
3448               errors.print("Directory " + encodedName + " duplicate??" +
3449                            hbi.getHdfsRegionDir());
3450             }
3451 
3452             he.hdfsRegionDir = regionDir.getPath();
3453             he.hdfsRegionDirModTime = regionDir.getModificationTime();
3454             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
3455             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
3456             // we add to orphan list when we attempt to read .regioninfo
3457 
3458             // Set a flag if this region contains only edits
3459             // This is special case if a region is left after split
3460             he.hdfsOnlyEdits = true;
3461             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3462             Path ePath = HLogUtil.getRegionDirRecoveredEditsDir(regionDir.getPath());
3463             for (FileStatus subDir : subDirs) {
3464               String sdName = subDir.getPath().getName();
3465               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3466                 he.hdfsOnlyEdits = false;
3467                 break;
3468               }
3469             }
3470             hbi.hdfsEntry = he;
3471           }
3472         }
3473       } catch (IOException e) {
3474         // unable to connect to the region server.
3475         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3476             + tableDir.getPath().getName()
3477             + " Unable to fetch region information. " + e);
3478         throw e;
3479       }
3480       return null;
3481     }
3482   }
3483 
3484   /**
3485    * Contact hdfs and get all information about specified table directory into
3486    * regioninfo list.
3487    */
3488   static class WorkItemHdfsRegionInfo implements Callable<Void> {
3489     private HbckInfo hbi;
3490     private HBaseFsck hbck;
3491     private ErrorReporter errors;
3492 
3493     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
3494       this.hbi = hbi;
3495       this.hbck = hbck;
3496       this.errors = errors;
3497     }
3498 
3499     @Override
3500     public synchronized Void call() throws IOException {
3501       // only load entries that haven't been loaded yet.
3502       if (hbi.getHdfsHRI() == null) {
3503         try {
3504           hbck.loadHdfsRegioninfo(hbi);
3505         } catch (IOException ioe) {
3506           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3507               + hbi.getTableName() + " in hdfs dir "
3508               + hbi.getHdfsRegionDir()
3509               + "!  It may be an invalid format or version file.  Treating as "
3510               + "an orphaned regiondir.";
3511           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3512           try {
3513             hbck.debugLsr(hbi.getHdfsRegionDir());
3514           } catch (IOException ioe2) {
3515             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3516             throw ioe2;
3517           }
3518           hbck.orphanHdfsDirs.add(hbi);
3519           throw ioe;
3520         }
3521       }
3522       return null;
3523     }
3524   };
3525 
3526   /**
3527    * Display the full report from fsck. This displays all live and dead region
3528    * servers, and all known regions.
3529    */
3530   public static void setDisplayFullReport() {
3531     details = true;
3532   }
3533 
3534   /**
3535    * Set summary mode.
3536    * Print only summary of the tables and status (OK or INCONSISTENT)
3537    */
3538   void setSummary() {
3539     summary = true;
3540   }
3541 
3542   /**
3543    * Set hbase:meta check mode.
3544    * Print only info about hbase:meta table deployment/state
3545    */
3546   void setCheckMetaOnly() {
3547     checkMetaOnly = true;
3548   }
3549 
3550   /**
3551    * Set region boundaries check mode.
3552    */
3553   void setRegionBoundariesCheck() {
3554     checkRegionBoundaries = true;
3555   }
3556 
3557   /**
3558    * Set table locks fix mode.
3559    * Delete table locks held for a long time
3560    */
3561   public void setFixTableLocks(boolean shouldFix) {
3562     fixTableLocks = shouldFix;
3563   }
3564 
3565   /**
3566    * Check if we should rerun fsck again. This checks if we've tried to
3567    * fix something and we should rerun fsck tool again.
3568    * Display the full report from fsck. This displays all live and dead
3569    * region servers, and all known regions.
3570    */
3571   void setShouldRerun() {
3572     rerun = true;
3573   }
3574 
3575   boolean shouldRerun() {
3576     return rerun;
3577   }
3578 
3579   /**
3580    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3581    * found by fsck utility.
3582    */
3583   public void setFixAssignments(boolean shouldFix) {
3584     fixAssignments = shouldFix;
3585   }
3586 
3587   boolean shouldFixAssignments() {
3588     return fixAssignments;
3589   }
3590 
3591   public void setFixMeta(boolean shouldFix) {
3592     fixMeta = shouldFix;
3593   }
3594 
3595   boolean shouldFixMeta() {
3596     return fixMeta;
3597   }
3598 
3599   public void setFixEmptyMetaCells(boolean shouldFix) {
3600     fixEmptyMetaCells = shouldFix;
3601   }
3602 
3603   boolean shouldFixEmptyMetaCells() {
3604     return fixEmptyMetaCells;
3605   }
3606 
3607   public void setCheckHdfs(boolean checking) {
3608     checkHdfs = checking;
3609   }
3610 
3611   boolean shouldCheckHdfs() {
3612     return checkHdfs;
3613   }
3614 
3615   public void setFixHdfsHoles(boolean shouldFix) {
3616     fixHdfsHoles = shouldFix;
3617   }
3618 
3619   boolean shouldFixHdfsHoles() {
3620     return fixHdfsHoles;
3621   }
3622 
3623   public void setFixTableOrphans(boolean shouldFix) {
3624     fixTableOrphans = shouldFix;
3625   }
3626 
3627   boolean shouldFixTableOrphans() {
3628     return fixTableOrphans;
3629   }
3630 
3631   public void setFixHdfsOverlaps(boolean shouldFix) {
3632     fixHdfsOverlaps = shouldFix;
3633   }
3634 
3635   boolean shouldFixHdfsOverlaps() {
3636     return fixHdfsOverlaps;
3637   }
3638 
3639   public void setFixHdfsOrphans(boolean shouldFix) {
3640     fixHdfsOrphans = shouldFix;
3641   }
3642 
3643   boolean shouldFixHdfsOrphans() {
3644     return fixHdfsOrphans;
3645   }
3646 
3647   public void setFixVersionFile(boolean shouldFix) {
3648     fixVersionFile = shouldFix;
3649   }
3650 
3651   public boolean shouldFixVersionFile() {
3652     return fixVersionFile;
3653   }
3654 
3655   public void setSidelineBigOverlaps(boolean sbo) {
3656     this.sidelineBigOverlaps = sbo;
3657   }
3658 
3659   public boolean shouldSidelineBigOverlaps() {
3660     return sidelineBigOverlaps;
3661   }
3662 
3663   public void setFixSplitParents(boolean shouldFix) {
3664     fixSplitParents = shouldFix;
3665   }
3666 
3667   boolean shouldFixSplitParents() {
3668     return fixSplitParents;
3669   }
3670 
3671   public void setFixReferenceFiles(boolean shouldFix) {
3672     fixReferenceFiles = shouldFix;
3673   }
3674 
3675   boolean shouldFixReferenceFiles() {
3676     return fixReferenceFiles;
3677   }
3678 
3679   public boolean shouldIgnorePreCheckPermission() {
3680     return ignorePreCheckPermission;
3681   }
3682 
3683   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
3684     this.ignorePreCheckPermission = ignorePreCheckPermission;
3685   }
3686 
3687   /**
3688    * @param mm maximum number of regions to merge into a single region.
3689    */
3690   public void setMaxMerge(int mm) {
3691     this.maxMerge = mm;
3692   }
3693 
3694   public int getMaxMerge() {
3695     return maxMerge;
3696   }
3697 
3698   public void setMaxOverlapsToSideline(int mo) {
3699     this.maxOverlapsToSideline = mo;
3700   }
3701 
3702   public int getMaxOverlapsToSideline() {
3703     return maxOverlapsToSideline;
3704   }
3705 
3706   /**
3707    * Only check/fix tables specified by the list,
3708    * Empty list means all tables are included.
3709    */
3710   boolean isTableIncluded(TableName table) {
3711     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
3712   }
3713 
3714   public void includeTable(TableName table) {
3715     tablesIncluded.add(table);
3716   }
3717 
3718   Set<TableName> getIncludedTables() {
3719     return new HashSet<TableName>(tablesIncluded);
3720   }
3721 
3722   /**
3723    * We are interested in only those tables that have not changed their state in
3724    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
3725    * @param seconds - the time in seconds
3726    */
3727   public void setTimeLag(long seconds) {
3728     timelag = seconds * 1000; // convert to milliseconds
3729   }
3730 
3731   /**
3732    *
3733    * @param sidelineDir - HDFS path to sideline data
3734    */
3735   public void setSidelineDir(String sidelineDir) {
3736     this.sidelineDir = new Path(sidelineDir);
3737   }
3738 
3739   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
3740     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
3741   }
3742 
3743   public HFileCorruptionChecker getHFilecorruptionChecker() {
3744     return hfcc;
3745   }
3746 
3747   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
3748     this.hfcc = hfcc;
3749   }
3750 
3751   public void setRetCode(int code) {
3752     this.retcode = code;
3753   }
3754 
3755   public int getRetCode() {
3756     return retcode;
3757   }
3758 
3759   protected HBaseFsck printUsageAndExit() {
3760     StringWriter sw = new StringWriter(2048);
3761     PrintWriter out = new PrintWriter(sw);
3762     out.println("Usage: fsck [opts] {only tables}");
3763     out.println(" where [opts] are:");
3764     out.println("   -help Display help options (this)");
3765     out.println("   -details Display full report of all regions.");
3766     out.println("   -timelag <timeInSeconds>  Process only regions that " +
3767                        " have not experienced any metadata updates in the last " +
3768                        " <timeInSeconds> seconds.");
3769     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
3770         " before checking if the fix worked if run with -fix");
3771     out.println("   -summary Print only summary of the tables and status.");
3772     out.println("   -metaonly Only check the state of the hbase:meta table.");
3773     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
3774     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
3775 
3776     out.println("");
3777     out.println("  Metadata Repair options: (expert features, use with caution!)");
3778     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
3779     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
3780     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
3781     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
3782         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
3783     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
3784     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
3785     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
3786     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
3787     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
3788     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
3789     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
3790     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
3791     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
3792     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
3793     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
3794     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
3795         + " (empty REGIONINFO_QUALIFIER rows)");
3796 
3797     out.println("");
3798     out.println("  Datafile Repair options: (expert features, use with caution!)");
3799     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
3800     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
3801 
3802     out.println("");
3803     out.println("  Metadata Repair shortcuts");
3804     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
3805         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
3806     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
3807 
3808     out.println("");
3809     out.println("  Table lock options");
3810     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
3811 
3812     out.flush();
3813     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
3814 
3815     setRetCode(-2);
3816     return this;
3817   }
3818 
3819   /**
3820    * Main program
3821    *
3822    * @param args
3823    * @throws Exception
3824    */
3825   public static void main(String[] args) throws Exception {
3826     // create a fsck object
3827     Configuration conf = HBaseConfiguration.create();
3828     Path hbasedir = FSUtils.getRootDir(conf);
3829     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
3830     FSUtils.setFsDefault(conf, new Path(defaultFs));
3831 
3832     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
3833     System.exit(ret);
3834   }
3835 
3836   /**
3837    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
3838    */
3839   static class HBaseFsckTool extends Configured implements Tool {
3840     HBaseFsckTool(Configuration conf) { super(conf); }
3841     @Override
3842     public int run(String[] args) throws Exception {
3843       HBaseFsck hbck = new HBaseFsck(getConf());
3844       hbck.exec(hbck.executor, args);
3845       return hbck.getRetCode();
3846     }
3847   };
3848   
3849 
3850   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
3851     ServiceException, InterruptedException {
3852     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
3853 
3854     boolean checkCorruptHFiles = false;
3855     boolean sidelineCorruptHFiles = false;
3856 
3857     // Process command-line args.
3858     for (int i = 0; i < args.length; i++) {
3859       String cmd = args[i];
3860       if (cmd.equals("-help") || cmd.equals("-h")) {
3861         return printUsageAndExit();
3862       } else if (cmd.equals("-details")) {
3863         setDisplayFullReport();
3864       } else if (cmd.equals("-timelag")) {
3865         if (i == args.length - 1) {
3866           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
3867           return printUsageAndExit();
3868         }
3869         try {
3870           long timelag = Long.parseLong(args[i+1]);
3871           setTimeLag(timelag);
3872         } catch (NumberFormatException e) {
3873           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
3874           return printUsageAndExit();
3875         }
3876         i++;
3877       } else if (cmd.equals("-sleepBeforeRerun")) {
3878         if (i == args.length - 1) {
3879           errors.reportError(ERROR_CODE.WRONG_USAGE,
3880             "HBaseFsck: -sleepBeforeRerun needs a value.");
3881           return printUsageAndExit();
3882         }
3883         try {
3884           sleepBeforeRerun = Long.parseLong(args[i+1]);
3885         } catch (NumberFormatException e) {
3886           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
3887           return printUsageAndExit();
3888         }
3889         i++;
3890       } else if (cmd.equals("-sidelineDir")) {
3891         if (i == args.length - 1) {
3892           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
3893           return printUsageAndExit();
3894         }
3895         i++;
3896         setSidelineDir(args[i]);
3897       } else if (cmd.equals("-fix")) {
3898         errors.reportError(ERROR_CODE.WRONG_USAGE,
3899           "This option is deprecated, please use  -fixAssignments instead.");
3900         setFixAssignments(true);
3901       } else if (cmd.equals("-fixAssignments")) {
3902         setFixAssignments(true);
3903       } else if (cmd.equals("-fixMeta")) {
3904         setFixMeta(true);
3905       } else if (cmd.equals("-noHdfsChecking")) {
3906         setCheckHdfs(false);
3907       } else if (cmd.equals("-fixHdfsHoles")) {
3908         setFixHdfsHoles(true);
3909       } else if (cmd.equals("-fixHdfsOrphans")) {
3910         setFixHdfsOrphans(true);
3911       } else if (cmd.equals("-fixTableOrphans")) {
3912         setFixTableOrphans(true);
3913       } else if (cmd.equals("-fixHdfsOverlaps")) {
3914         setFixHdfsOverlaps(true);
3915       } else if (cmd.equals("-fixVersionFile")) {
3916         setFixVersionFile(true);
3917       } else if (cmd.equals("-sidelineBigOverlaps")) {
3918         setSidelineBigOverlaps(true);
3919       } else if (cmd.equals("-fixSplitParents")) {
3920         setFixSplitParents(true);
3921       } else if (cmd.equals("-ignorePreCheckPermission")) {
3922         setIgnorePreCheckPermission(true);
3923       } else if (cmd.equals("-checkCorruptHFiles")) {
3924         checkCorruptHFiles = true;
3925       } else if (cmd.equals("-sidelineCorruptHFiles")) {
3926         sidelineCorruptHFiles = true;
3927       } else if (cmd.equals("-fixReferenceFiles")) {
3928         setFixReferenceFiles(true);
3929       } else if (cmd.equals("-fixEmptyMetaCells")) {
3930         setFixEmptyMetaCells(true);
3931       } else if (cmd.equals("-repair")) {
3932         // this attempts to merge overlapping hdfs regions, needs testing
3933         // under load
3934         setFixHdfsHoles(true);
3935         setFixHdfsOrphans(true);
3936         setFixMeta(true);
3937         setFixAssignments(true);
3938         setFixHdfsOverlaps(true);
3939         setFixVersionFile(true);
3940         setSidelineBigOverlaps(true);
3941         setFixSplitParents(false);
3942         setCheckHdfs(true);
3943         setFixReferenceFiles(true);
3944         setFixTableLocks(true);
3945       } else if (cmd.equals("-repairHoles")) {
3946         // this will make all missing hdfs regions available but may lose data
3947         setFixHdfsHoles(true);
3948         setFixHdfsOrphans(false);
3949         setFixMeta(true);
3950         setFixAssignments(true);
3951         setFixHdfsOverlaps(false);
3952         setSidelineBigOverlaps(false);
3953         setFixSplitParents(false);
3954         setCheckHdfs(true);
3955       } else if (cmd.equals("-maxOverlapsToSideline")) {
3956         if (i == args.length - 1) {
3957           errors.reportError(ERROR_CODE.WRONG_USAGE,
3958             "-maxOverlapsToSideline needs a numeric value argument.");
3959           return printUsageAndExit();
3960         }
3961         try {
3962           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
3963           setMaxOverlapsToSideline(maxOverlapsToSideline);
3964         } catch (NumberFormatException e) {
3965           errors.reportError(ERROR_CODE.WRONG_USAGE,
3966             "-maxOverlapsToSideline needs a numeric value argument.");
3967           return printUsageAndExit();
3968         }
3969         i++;
3970       } else if (cmd.equals("-maxMerge")) {
3971         if (i == args.length - 1) {
3972           errors.reportError(ERROR_CODE.WRONG_USAGE,
3973             "-maxMerge needs a numeric value argument.");
3974           return printUsageAndExit();
3975         }
3976         try {
3977           int maxMerge = Integer.parseInt(args[i+1]);
3978           setMaxMerge(maxMerge);
3979         } catch (NumberFormatException e) {
3980           errors.reportError(ERROR_CODE.WRONG_USAGE,
3981             "-maxMerge needs a numeric value argument.");
3982           return printUsageAndExit();
3983         }
3984         i++;
3985       } else if (cmd.equals("-summary")) {
3986         setSummary();
3987       } else if (cmd.equals("-metaonly")) {
3988         setCheckMetaOnly();
3989       } else if (cmd.equals("-boundaries")) {
3990         setRegionBoundariesCheck();
3991       } else if (cmd.equals("-fixTableLocks")) {
3992         setFixTableLocks(true);
3993       } else if (cmd.startsWith("-")) {
3994         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
3995         return printUsageAndExit();
3996       } else {
3997         includeTable(TableName.valueOf(cmd));
3998         errors.print("Allow checking/fixes for table: " + cmd);
3999       }
4000     }
4001 
4002     // pre-check current user has FS write permission or not
4003     try {
4004       preCheckPermission();
4005     } catch (AccessControlException ace) {
4006       Runtime.getRuntime().exit(-1);
4007     } catch (IOException ioe) {
4008       Runtime.getRuntime().exit(-1);
4009     }
4010 
4011     // do the real work of hbck
4012     connect();
4013 
4014     try {
4015       // if corrupt file mode is on, first fix them since they may be opened later
4016       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4017         LOG.info("Checking all hfiles for corruption");
4018         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4019         setHFileCorruptionChecker(hfcc); // so we can get result
4020         Collection<TableName> tables = getIncludedTables();
4021         Collection<Path> tableDirs = new ArrayList<Path>();
4022         Path rootdir = FSUtils.getRootDir(getConf());
4023         if (tables.size() > 0) {
4024           for (TableName t : tables) {
4025             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4026           }
4027         } else {
4028           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4029         }
4030         hfcc.checkTables(tableDirs);
4031         hfcc.report(errors);
4032       }
4033 
4034       // check and fix table integrity, region consistency.
4035       int code = onlineHbck();
4036       setRetCode(code);
4037       // If we have changed the HBase state it is better to run hbck again
4038       // to see if we haven't broken something else in the process.
4039       // We run it only once more because otherwise we can easily fall into
4040       // an infinite loop.
4041       if (shouldRerun()) {
4042         try {
4043           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4044           Thread.sleep(sleepBeforeRerun);
4045         } catch (InterruptedException ie) {
4046           LOG.warn("Interrupted while sleeping");
4047           return this;
4048         }
4049         // Just report
4050         setFixAssignments(false);
4051         setFixMeta(false);
4052         setFixHdfsHoles(false);
4053         setFixHdfsOverlaps(false);
4054         setFixVersionFile(false);
4055         setFixTableOrphans(false);
4056         errors.resetErrors();
4057         code = onlineHbck();
4058         setRetCode(code);
4059       }
4060     } finally {
4061       IOUtils.cleanup(null, connection, meta, admin);
4062     }
4063     return this;
4064   }
4065 
4066   /**
4067    * ls -r for debugging purposes
4068    */
4069   void debugLsr(Path p) throws IOException {
4070     debugLsr(getConf(), p, errors);
4071   }
4072 
4073   /**
4074    * ls -r for debugging purposes
4075    */
4076   public static void debugLsr(Configuration conf,
4077       Path p) throws IOException {
4078     debugLsr(conf, p, new PrintingErrorReporter());
4079   }
4080 
4081   /**
4082    * ls -r for debugging purposes
4083    */
4084   public static void debugLsr(Configuration conf,
4085       Path p, ErrorReporter errors) throws IOException {
4086     if (!LOG.isDebugEnabled() || p == null) {
4087       return;
4088     }
4089     FileSystem fs = p.getFileSystem(conf);
4090 
4091     if (!fs.exists(p)) {
4092       // nothing
4093       return;
4094     }
4095     errors.print(p.toString());
4096 
4097     if (fs.isFile(p)) {
4098       return;
4099     }
4100 
4101     if (fs.getFileStatus(p).isDir()) {
4102       FileStatus[] fss= fs.listStatus(p);
4103       for (FileStatus status : fss) {
4104         debugLsr(conf, status.getPath(), errors);
4105       }
4106     }
4107   }
4108 }