View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.Closeable;
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.PrintWriter;
24  import java.io.StringWriter;
25  import java.net.InetAddress;
26  import java.net.URI;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Collection;
30  import java.util.Collections;
31  import java.util.Comparator;
32  import java.util.HashMap;
33  import java.util.HashSet;
34  import java.util.Iterator;
35  import java.util.List;
36  import java.util.Map;
37  import java.util.Map.Entry;
38  import java.util.Set;
39  import java.util.SortedMap;
40  import java.util.SortedSet;
41  import java.util.TreeMap;
42  import java.util.TreeSet;
43  import java.util.concurrent.Callable;
44  import java.util.concurrent.ConcurrentSkipListMap;
45  import java.util.concurrent.ExecutionException;
46  import java.util.concurrent.ExecutorService;
47  import java.util.concurrent.Executors;
48  import java.util.concurrent.Future;
49  import java.util.concurrent.FutureTask;
50  import java.util.concurrent.ScheduledThreadPoolExecutor;
51  import java.util.concurrent.TimeUnit;
52  import java.util.concurrent.TimeoutException;
53  import java.util.concurrent.atomic.AtomicBoolean;
54  import java.util.concurrent.atomic.AtomicInteger;
55  
56  import com.google.common.base.Joiner;
57  import com.google.common.base.Preconditions;
58  import com.google.common.collect.Lists;
59  import com.google.common.collect.Multimap;
60  import com.google.common.collect.TreeMultimap;
61  import com.google.protobuf.ServiceException;
62  import org.apache.commons.lang.RandomStringUtils;
63  import org.apache.commons.lang.StringUtils;
64  import org.apache.commons.logging.Log;
65  import org.apache.commons.logging.LogFactory;
66  import org.apache.hadoop.conf.Configuration;
67  import org.apache.hadoop.conf.Configured;
68  import org.apache.hadoop.fs.FSDataOutputStream;
69  import org.apache.hadoop.fs.FileStatus;
70  import org.apache.hadoop.fs.FileSystem;
71  import org.apache.hadoop.fs.Path;
72  import org.apache.hadoop.fs.permission.FsAction;
73  import org.apache.hadoop.fs.permission.FsPermission;
74  import org.apache.hadoop.hbase.Abortable;
75  import org.apache.hadoop.hbase.Cell;
76  import org.apache.hadoop.hbase.ClusterStatus;
77  import org.apache.hadoop.hbase.HBaseConfiguration;
78  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
79  import org.apache.hadoop.hbase.HColumnDescriptor;
80  import org.apache.hadoop.hbase.HConstants;
81  import org.apache.hadoop.hbase.HRegionInfo;
82  import org.apache.hadoop.hbase.HRegionLocation;
83  import org.apache.hadoop.hbase.HTableDescriptor;
84  import org.apache.hadoop.hbase.KeyValue;
85  import org.apache.hadoop.hbase.MasterNotRunningException;
86  import org.apache.hadoop.hbase.MetaTableAccessor;
87  import org.apache.hadoop.hbase.RegionLocations;
88  import org.apache.hadoop.hbase.ServerName;
89  import org.apache.hadoop.hbase.TableDescriptor;
90  import org.apache.hadoop.hbase.TableName;
91  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
92  import org.apache.hadoop.hbase.classification.InterfaceAudience;
93  import org.apache.hadoop.hbase.classification.InterfaceStability;
94  import org.apache.hadoop.hbase.client.Admin;
95  import org.apache.hadoop.hbase.client.ClusterConnection;
96  import org.apache.hadoop.hbase.client.ConnectionFactory;
97  import org.apache.hadoop.hbase.client.Delete;
98  import org.apache.hadoop.hbase.client.Get;
99  import org.apache.hadoop.hbase.client.HBaseAdmin;
100 import org.apache.hadoop.hbase.client.HConnection;
101 import org.apache.hadoop.hbase.client.MetaScanner;
102 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
103 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
104 import org.apache.hadoop.hbase.client.Put;
105 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
106 import org.apache.hadoop.hbase.client.Result;
107 import org.apache.hadoop.hbase.client.RowMutations;
108 import org.apache.hadoop.hbase.client.Table;
109 import org.apache.hadoop.hbase.client.TableState;
110 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
111 import org.apache.hadoop.hbase.io.hfile.HFile;
112 import org.apache.hadoop.hbase.master.MasterFileSystem;
113 import org.apache.hadoop.hbase.master.RegionState;
114 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
115 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
116 import org.apache.hadoop.hbase.regionserver.HRegion;
117 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
118 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
119 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
120 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
121 import org.apache.hadoop.hbase.security.AccessDeniedException;
122 import org.apache.hadoop.hbase.security.UserProvider;
123 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
124 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
125 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
126 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
127 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
128 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
129 import org.apache.hadoop.hbase.wal.WAL;
130 import org.apache.hadoop.hbase.wal.WALFactory;
131 import org.apache.hadoop.hbase.wal.WALSplitter;
132 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
133 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
134 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
135 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
136 import org.apache.hadoop.io.IOUtils;
137 import org.apache.hadoop.ipc.RemoteException;
138 import org.apache.hadoop.security.UserGroupInformation;
139 import org.apache.hadoop.util.ReflectionUtils;
140 import org.apache.hadoop.util.Tool;
141 import org.apache.hadoop.util.ToolRunner;
142 import org.apache.zookeeper.KeeperException;
143 
144 /**
145  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
146  * table integrity problems in a corrupted HBase.
147  * <p>
148  * Region consistency checks verify that hbase:meta, region deployment on region
149  * servers and the state of data in HDFS (.regioninfo files) all are in
150  * accordance.
151  * <p>
152  * Table integrity checks verify that all possible row keys resolve to exactly
153  * one region of a table.  This means there are no individual degenerate
154  * or backwards regions; no holes between regions; and that there are no
155  * overlapping regions.
156  * <p>
157  * The general repair strategy works in two phases:
158  * <ol>
159  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
160  * <li> Repair Region Consistency with hbase:meta and assignments
161  * </ol>
162  * <p>
163  * For table integrity repairs, the tables' region directories are scanned
164  * for .regioninfo files.  Each table's integrity is then verified.  If there
165  * are any orphan regions (regions with no .regioninfo files) or holes, new
166  * regions are fabricated.  Backwards regions are sidelined as well as empty
167  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
168  * a new region is created and all data is merged into the new region.
169  * <p>
170  * Table integrity repairs deal solely with HDFS and could potentially be done
171  * offline -- the hbase region servers or master do not need to be running.
172  * This phase can eventually be used to completely reconstruct the hbase:meta table in
173  * an offline fashion.
174  * <p>
175  * Region consistency requires three conditions -- 1) valid .regioninfo file
176  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
177  * and 3) a region is deployed only at the regionserver that was assigned to
178  * with proper state in the master.
179  * <p>
180  * Region consistency repairs require hbase to be online so that hbck can
181  * contact the HBase master and region servers.  The hbck#connect() method must
182  * first be called successfully.  Much of the region consistency information
183  * is transient and less risky to repair.
184  * <p>
185  * If hbck is run from the command line, there are a handful of arguments that
186  * can be used to limit the kinds of repairs hbck will do.  See the code in
187  * {@link #printUsageAndExit()} for more details.
188  */
189 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
190 @InterfaceStability.Evolving
191 public class HBaseFsck extends Configured implements Closeable {
192   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
193   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
194   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
195   private static boolean rsSupportsOffline = true;
196   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
197   private static final int DEFAULT_MAX_MERGE = 5;
198   private static final String TO_BE_LOADED = "to_be_loaded";
199   private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
200 
201 
202   /**********************
203    * Internal resources
204    **********************/
205   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
206   private ClusterStatus status;
207   private ClusterConnection connection;
208   private Admin admin;
209   private Table meta;
210   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
211   protected ExecutorService executor;
212   private long startMillis = System.currentTimeMillis();
213   private HFileCorruptionChecker hfcc;
214   private int retcode = 0;
215   private Path HBCK_LOCK_PATH;
216   private FSDataOutputStream hbckOutFd;
217   // This lock is to prevent cleanup of balancer resources twice between
218   // ShutdownHook and the main code. We cleanup only if the connect() is
219   // successful
220   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
221 
222   /***********
223    * Options
224    ***********/
225   private static boolean details = false; // do we display the full report
226   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
227   private boolean fixAssignments = false; // fix assignment errors?
228   private boolean fixMeta = false; // fix meta errors?
229   private boolean checkHdfs = true; // load and check fs consistency?
230   private boolean fixHdfsHoles = false; // fix fs holes?
231   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
232   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
233   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
234   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
235   private boolean fixSplitParents = false; // fix lingering split parents
236   private boolean fixReferenceFiles = false; // fix lingering reference store file
237   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
238   private boolean fixTableLocks = false; // fix table locks which are expired
239   private boolean fixAny = false; // Set to true if any of the fix is required.
240 
241   // limit checking/fixes to listed tables, if empty attempt to check/fix all
242   // hbase:meta are always checked
243   private Set<TableName> tablesIncluded = new HashSet<TableName>();
244   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
245   // maximum number of overlapping regions to sideline
246   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
247   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
248   private Path sidelineDir = null;
249 
250   private boolean rerun = false; // if we tried to fix something, rerun hbck
251   private static boolean summary = false; // if we want to print less output
252   private boolean checkMetaOnly = false;
253   private boolean checkRegionBoundaries = false;
254   private boolean ignorePreCheckPermission = false; // if pre-check permission
255 
256   /*********
257    * State
258    *********/
259   final private ErrorReporter errors;
260   int fixes = 0;
261 
262   /**
263    * This map contains the state of all hbck items.  It maps from encoded region
264    * name to HbckInfo structure.  The information contained in HbckInfo is used
265    * to detect and correct consistency (hdfs/meta/deployment) problems.
266    */
267   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
268   // Empty regioninfo qualifiers in hbase:meta
269   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
270 
271   /**
272    * This map from Tablename -> TableInfo contains the structures necessary to
273    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
274    * to prevent dupes.
275    *
276    * If tablesIncluded is empty, this map contains all tables.
277    * Otherwise, it contains only meta tables and tables in tablesIncluded,
278    * unless checkMetaOnly is specified, in which case, it contains only
279    * the meta table
280    */
281   private SortedMap<TableName, TableInfo> tablesInfo =
282       new ConcurrentSkipListMap<TableName, TableInfo>();
283 
284   /**
285    * When initially looking at HDFS, we attempt to find any orphaned data.
286    */
287   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
288 
289   private Map<TableName, Set<String>> orphanTableDirs =
290       new HashMap<TableName, Set<String>>();
291   private Map<TableName, TableState> tableStates =
292       new HashMap<TableName, TableState>();
293 
294   /**
295    * Constructor
296    *
297    * @param conf Configuration object
298    * @throws MasterNotRunningException if the master is not running
299    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
300    */
301   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
302       ZooKeeperConnectionException, IOException, ClassNotFoundException {
303     super(conf);
304     // make a copy, just to be sure we're not overriding someone else's config
305     setConf(HBaseConfiguration.create(getConf()));
306     // disable blockcache for tool invocation, see HBASE-10500
307     getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
308     // Disable usage of meta replicas in hbck
309     getConf().setBoolean(HConstants.USE_META_REPLICAS, false);
310     errors = getErrorReporter(conf);
311 
312     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
313     executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
314   }
315 
316   /**
317    * Constructor
318    *
319    * @param conf
320    *          Configuration object
321    * @throws MasterNotRunningException
322    *           if the master is not running
323    * @throws ZooKeeperConnectionException
324    *           if unable to connect to ZooKeeper
325    */
326   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
327       ZooKeeperConnectionException, IOException, ClassNotFoundException {
328     super(conf);
329     errors = getErrorReporter(getConf());
330     this.executor = exec;
331   }
332   
333   private class FileLockCallable implements Callable<FSDataOutputStream> {
334     @Override
335     public FSDataOutputStream call() throws IOException {
336       try {
337         FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
338         FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
339             HConstants.DATA_FILE_UMASK_KEY);
340         Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
341         fs.mkdirs(tmpDir);
342         HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
343         final FSDataOutputStream out = FSUtils.create(fs, HBCK_LOCK_PATH, defaultPerms, false);
344         out.writeBytes(InetAddress.getLocalHost().toString());
345         out.flush();
346         return out;
347       } catch(RemoteException e) {
348         if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
349           return null;
350         } else {
351           throw e;
352         }
353       }
354     }
355   }
356 
357   /**
358    * This method maintains a lock using a file. If the creation fails we return null
359    *
360    * @return FSDataOutputStream object corresponding to the newly opened lock file
361    * @throws IOException
362    */
363   private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
364     FileLockCallable callable = new FileLockCallable();
365     ExecutorService executor = Executors.newFixedThreadPool(1);
366     FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
367     executor.execute(futureTask);
368     final int timeoutInSeconds = 30;
369     FSDataOutputStream stream = null;
370     try {
371       stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
372     } catch (ExecutionException ee) {
373       LOG.warn("Encountered exception when opening lock file", ee);
374     } catch (InterruptedException ie) {
375       LOG.warn("Interrupted when opening lock file", ie);
376       Thread.currentThread().interrupt();
377     } catch (TimeoutException exception) {
378       // took too long to obtain lock
379       LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
380       futureTask.cancel(true);
381     } finally {
382       executor.shutdownNow();
383     }
384     return stream;
385   }
386 
387   private void unlockHbck() {
388     if(hbckLockCleanup.compareAndSet(true, false)){
389       IOUtils.closeStream(hbckOutFd);
390       try{
391         FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true);
392       } catch(IOException ioe) {
393         LOG.warn("Failed to delete " + HBCK_LOCK_PATH);
394         LOG.debug(ioe);
395       }
396     }
397   }
398 
399   /**
400    * To repair region consistency, one must call connect() in order to repair
401    * online state.
402    */
403   public void connect() throws IOException {
404 
405     // Check if another instance of balancer is running
406     hbckOutFd = checkAndMarkRunningHbck();
407     if (hbckOutFd == null) {
408       setRetCode(-1);
409       LOG.error("Another instance of hbck is running, exiting this instance.[If you are sure" +
410           " no other instance is running, delete the lock file " +
411           HBCK_LOCK_PATH + " and rerun the tool]");
412       throw new IOException("Duplicate hbck - Abort");
413     }
414 
415     // Make sure to cleanup the lock
416     hbckLockCleanup.set(true);
417 
418     // Add a shutdown hook to this thread, incase user tries to
419     // kill the hbck with a ctrl-c, we want to cleanup the lock so that
420     // it is available for further calls
421     Runtime.getRuntime().addShutdownHook(new Thread() {
422       @Override
423       public void run() {
424         IOUtils.closeStream(HBaseFsck.this);
425         unlockHbck();
426       }
427     });
428     LOG.debug("Launching hbck");
429 
430     connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
431     admin = connection.getAdmin();
432     meta = connection.getTable(TableName.META_TABLE_NAME);
433     status = admin.getClusterStatus();
434   }
435 
436   /**
437    * Get deployed regions according to the region servers.
438    */
439   private void loadDeployedRegions() throws IOException, InterruptedException {
440     // From the master, get a list of all known live region servers
441     Collection<ServerName> regionServers = status.getServers();
442     errors.print("Number of live region servers: " + regionServers.size());
443     if (details) {
444       for (ServerName rsinfo: regionServers) {
445         errors.print("  " + rsinfo.getServerName());
446       }
447     }
448 
449     // From the master, get a list of all dead region servers
450     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
451     errors.print("Number of dead region servers: " + deadRegionServers.size());
452     if (details) {
453       for (ServerName name: deadRegionServers) {
454         errors.print("  " + name);
455       }
456     }
457 
458     // Print the current master name and state
459     errors.print("Master: " + status.getMaster());
460 
461     // Print the list of all backup masters
462     Collection<ServerName> backupMasters = status.getBackupMasters();
463     errors.print("Number of backup masters: " + backupMasters.size());
464     if (details) {
465       for (ServerName name: backupMasters) {
466         errors.print("  " + name);
467       }
468     }
469 
470     errors.print("Average load: " + status.getAverageLoad());
471     errors.print("Number of requests: " + status.getRequestsCount());
472     errors.print("Number of regions: " + status.getRegionsCount());
473 
474     Map<String, RegionState> rits = status.getRegionsInTransition();
475     errors.print("Number of regions in transition: " + rits.size());
476     if (details) {
477       for (RegionState state: rits.values()) {
478         errors.print("  " + state.toDescriptiveString());
479       }
480     }
481 
482     // Determine what's deployed
483     processRegionServers(regionServers);
484   }
485 
486   /**
487    * Clear the current state of hbck.
488    */
489   private void clearState() {
490     // Make sure regionInfo is empty before starting
491     fixes = 0;
492     regionInfoMap.clear();
493     emptyRegionInfoQualifiers.clear();
494     tableStates.clear();
495     errors.clear();
496     tablesInfo.clear();
497     orphanHdfsDirs.clear();
498   }
499 
500   /**
501    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
502    * the table integrity rules.  HBase doesn't need to be online for this
503    * operation to work.
504    */
505   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
506     // Initial pass to fix orphans.
507     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
508         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
509       LOG.info("Loading regioninfos HDFS");
510       // if nothing is happening this should always complete in two iterations.
511       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
512       int curIter = 0;
513       do {
514         clearState(); // clears hbck state and reset fixes to 0 and.
515         // repair what's on HDFS
516         restoreHdfsIntegrity();
517         curIter++;// limit the number of iterations.
518       } while (fixes > 0 && curIter <= maxIterations);
519 
520       // Repairs should be done in the first iteration and verification in the second.
521       // If there are more than 2 passes, something funny has happened.
522       if (curIter > 2) {
523         if (curIter == maxIterations) {
524           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
525               + "Tables integrity may not be fully repaired!");
526         } else {
527           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
528         }
529       }
530     }
531   }
532 
533   /**
534    * This repair method requires the cluster to be online since it contacts
535    * region servers and the masters.  It makes each region's state in HDFS, in
536    * hbase:meta, and deployments consistent.
537    *
538    * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable
539    * error.  If 0, we have a clean hbase.
540    */
541   public int onlineConsistencyRepair() throws IOException, KeeperException,
542     InterruptedException {
543     clearState();
544 
545     // get regions according to what is online on each RegionServer
546     loadDeployedRegions();
547     // check whether hbase:meta is deployed and online
548     recordMetaRegion();
549     // Check if hbase:meta is found only once and in the right place
550     if (!checkMetaRegion()) {
551       String errorMsg = "hbase:meta table is not consistent. ";
552       if (shouldFixAssignments()) {
553         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
554       } else {
555         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
556       }
557       errors.reportError(errorMsg + " Exiting...");
558       return -2;
559     }
560     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
561     LOG.info("Loading regionsinfo from the hbase:meta table");
562     boolean success = loadMetaEntries();
563     if (!success) return -1;
564 
565     // Empty cells in hbase:meta?
566     reportEmptyMetaCells();
567 
568     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
569     if (shouldFixEmptyMetaCells()) {
570       fixEmptyMetaCells();
571     }
572 
573     // get a list of all tables that have not changed recently.
574     if (!checkMetaOnly) {
575       reportTablesInFlux();
576     }
577 
578     // Get disabled tables states
579     loadTableStates();
580 
581     // load regiondirs and regioninfos from HDFS
582     if (shouldCheckHdfs()) {
583       loadHdfsRegionDirs();
584       loadHdfsRegionInfos();
585     }
586 
587     // fix the orphan tables
588     fixOrphanTables();
589 
590     // Check and fix consistency
591     checkAndFixConsistency();
592 
593     // Check integrity (does not fix)
594     checkIntegrity();
595     return errors.getErrorList().size();
596   }
597 
598   /**
599    * Contacts the master and prints out cluster-wide information
600    * @return 0 on success, non-zero on failure
601    */
602   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
603     // print hbase server version
604     errors.print("Version: " + status.getHBaseVersion());
605     offlineHdfsIntegrityRepair();
606 
607     // turn the balancer off
608     boolean oldBalancer = admin.setBalancerRunning(false, true);
609     try {
610       onlineConsistencyRepair();
611     }
612     finally {
613       admin.setBalancerRunning(oldBalancer, false);
614     }
615 
616     if (checkRegionBoundaries) {
617       checkRegionBoundaries();
618     }
619 
620     offlineReferenceFileRepair();
621 
622     checkAndFixTableLocks();
623 
624     // Remove the hbck lock
625     unlockHbck();
626 
627     // Print table summary
628     printTableSummary(tablesInfo);
629     return errors.summarize();
630   }
631 
632   public static byte[] keyOnly (byte[] b) {
633     if (b == null)
634       return b;
635     int rowlength = Bytes.toShort(b, 0);
636     byte[] result = new byte[rowlength];
637     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
638     return result;
639   }
640 
641   @Override
642   public void close() throws IOException {
643     IOUtils.cleanup(null, admin, meta, connection);
644   }
645 
646   private static class RegionBoundariesInformation {
647     public byte [] regionName;
648     public byte [] metaFirstKey;
649     public byte [] metaLastKey;
650     public byte [] storesFirstKey;
651     public byte [] storesLastKey;
652     @Override
653     public String toString () {
654       return "regionName=" + Bytes.toStringBinary(regionName) +
655              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
656              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
657              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
658              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
659     }
660   }
661 
662   public void checkRegionBoundaries() {
663     try {
664       ByteArrayComparator comparator = new ByteArrayComparator();
665       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), connection, false);
666       final RegionBoundariesInformation currentRegionBoundariesInformation =
667           new RegionBoundariesInformation();
668       Path hbaseRoot = FSUtils.getRootDir(getConf());
669       for (HRegionInfo regionInfo : regions) {
670         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
671         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
672         // For each region, get the start and stop key from the META and compare them to the
673         // same information from the Stores.
674         Path path = new Path(tableDir, regionInfo.getEncodedName());
675         FileSystem fs = path.getFileSystem(getConf());
676         FileStatus[] files = fs.listStatus(path);
677         // For all the column families in this region...
678         byte[] storeFirstKey = null;
679         byte[] storeLastKey = null;
680         for (FileStatus file : files) {
681           String fileName = file.getPath().toString();
682           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
683           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
684             FileStatus[] storeFiles = fs.listStatus(file.getPath());
685             // For all the stores in this column family.
686             for (FileStatus storeFile : storeFiles) {
687               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
688                   getConf()), getConf());
689               if ((reader.getFirstKey() != null)
690                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
691                       reader.getFirstKey()) > 0))) {
692                 storeFirstKey = reader.getFirstKey();
693               }
694               if ((reader.getLastKey() != null)
695                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
696                       reader.getLastKey())) < 0)) {
697                 storeLastKey = reader.getLastKey();
698               }
699               reader.close();
700             }
701           }
702         }
703         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
704         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
705         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
706         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
707         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
708           currentRegionBoundariesInformation.metaFirstKey = null;
709         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
710           currentRegionBoundariesInformation.metaLastKey = null;
711 
712         // For a region to be correct, we need the META start key to be smaller or equal to the
713         // smallest start key from all the stores, and the start key from the next META entry to
714         // be bigger than the last key from all the current stores. First region start key is null;
715         // Last region end key is null; some regions can be empty and not have any store.
716 
717         boolean valid = true;
718         // Checking start key.
719         if ((currentRegionBoundariesInformation.storesFirstKey != null)
720             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
721           valid = valid
722               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
723                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
724         }
725         // Checking stop key.
726         if ((currentRegionBoundariesInformation.storesLastKey != null)
727             && (currentRegionBoundariesInformation.metaLastKey != null)) {
728           valid = valid
729               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
730                 currentRegionBoundariesInformation.metaLastKey) < 0;
731         }
732         if (!valid) {
733           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
734             tablesInfo.get(regionInfo.getTable()));
735           LOG.warn("Region's boundaries not alligned between stores and META for:");
736           LOG.warn(currentRegionBoundariesInformation);
737         }
738       }
739     } catch (IOException e) {
740       LOG.error(e);
741     }
742   }
743 
744   /**
745    * Iterates through the list of all orphan/invalid regiondirs.
746    */
747   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
748     for (HbckInfo hi : orphanHdfsDirs) {
749       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
750       adoptHdfsOrphan(hi);
751     }
752   }
753 
754   /**
755    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
756    * these orphans by creating a new region, and moving the column families,
757    * recovered edits, WALs, into the new region dir.  We determine the region
758    * startkey and endkeys by looking at all of the hfiles inside the column
759    * families to identify the min and max keys. The resulting region will
760    * likely violate table integrity but will be dealt with by merging
761    * overlapping regions.
762    */
763   @SuppressWarnings("deprecation")
764   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
765     Path p = hi.getHdfsRegionDir();
766     FileSystem fs = p.getFileSystem(getConf());
767     FileStatus[] dirs = fs.listStatus(p);
768     if (dirs == null) {
769       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
770           p + ". This dir could probably be deleted.");
771       return ;
772     }
773 
774     TableName tableName = hi.getTableName();
775     TableInfo tableInfo = tablesInfo.get(tableName);
776     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
777     HTableDescriptor template = tableInfo.getHTD();
778 
779     // find min and max key values
780     Pair<byte[],byte[]> orphanRegionRange = null;
781     for (FileStatus cf : dirs) {
782       String cfName= cf.getPath().getName();
783       // TODO Figure out what the special dirs are
784       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
785 
786       FileStatus[] hfiles = fs.listStatus(cf.getPath());
787       for (FileStatus hfile : hfiles) {
788         byte[] start, end;
789         HFile.Reader hf = null;
790         try {
791           CacheConfig cacheConf = new CacheConfig(getConf());
792           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
793           hf.loadFileInfo();
794           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
795           start = startKv.getRow();
796           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
797           end = endKv.getRow();
798         } catch (IOException ioe) {
799           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
800           continue;
801         } catch (NullPointerException ioe) {
802           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
803           continue;
804         } finally {
805           if (hf != null) {
806             hf.close();
807           }
808         }
809 
810         // expand the range to include the range of all hfiles
811         if (orphanRegionRange == null) {
812           // first range
813           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
814         } else {
815           // TODO add test
816 
817           // expand range only if the hfile is wider.
818           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
819             orphanRegionRange.setFirst(start);
820           }
821           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
822             orphanRegionRange.setSecond(end);
823           }
824         }
825       }
826     }
827     if (orphanRegionRange == null) {
828       LOG.warn("No data in dir " + p + ", sidelining data");
829       fixes++;
830       sidelineRegionDir(fs, hi);
831       return;
832     }
833     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
834         Bytes.toString(orphanRegionRange.getSecond()) + ")");
835 
836     // create new region on hdfs.  move data into place.
837     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
838     LOG.info("Creating new region : " + hri);
839     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
840     Path target = region.getRegionFileSystem().getRegionDir();
841 
842     // rename all the data to new region
843     mergeRegionDirs(target, hi);
844     fixes++;
845   }
846 
847   /**
848    * This method determines if there are table integrity errors in HDFS.  If
849    * there are errors and the appropriate "fix" options are enabled, the method
850    * will first correct orphan regions making them into legit regiondirs, and
851    * then reload to merge potentially overlapping regions.
852    *
853    * @return number of table integrity errors found
854    */
855   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
856     // Determine what's on HDFS
857     LOG.info("Loading HBase regioninfo from HDFS...");
858     loadHdfsRegionDirs(); // populating regioninfo table.
859 
860     int errs = errors.getErrorList().size();
861     // First time just get suggestions.
862     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
863     checkHdfsIntegrity(false, false);
864 
865     if (errors.getErrorList().size() == errs) {
866       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
867       return 0;
868     }
869 
870     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
871       adoptHdfsOrphans(orphanHdfsDirs);
872       // TODO optimize by incrementally adding instead of reloading.
873     }
874 
875     // Make sure there are no holes now.
876     if (shouldFixHdfsHoles()) {
877       clearState(); // this also resets # fixes.
878       loadHdfsRegionDirs();
879       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
880       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
881     }
882 
883     // Now we fix overlaps
884     if (shouldFixHdfsOverlaps()) {
885       // second pass we fix overlaps.
886       clearState(); // this also resets # fixes.
887       loadHdfsRegionDirs();
888       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
889       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
890     }
891 
892     return errors.getErrorList().size();
893   }
894 
895   /**
896    * Scan all the store file names to find any lingering reference files,
897    * which refer to some none-exiting files. If "fix" option is enabled,
898    * any lingering reference file will be sidelined if found.
899    * <p>
900    * Lingering reference file prevents a region from opening. It has to
901    * be fixed before a cluster can start properly.
902    */
903   private void offlineReferenceFileRepair() throws IOException {
904     Configuration conf = getConf();
905     Path hbaseRoot = FSUtils.getRootDir(conf);
906     FileSystem fs = hbaseRoot.getFileSystem(conf);
907     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
908     for (Path path: allFiles.values()) {
909       boolean isReference = false;
910       try {
911         isReference = StoreFileInfo.isReference(path);
912       } catch (Throwable t) {
913         // Ignore. Some files may not be store files at all.
914         // For example, files under .oldlogs folder in hbase:meta
915         // Warning message is already logged by
916         // StoreFile#isReference.
917       }
918       if (!isReference) continue;
919 
920       Path referredToFile = StoreFileInfo.getReferredToFile(path);
921       if (fs.exists(referredToFile)) continue;  // good, expected
922 
923       // Found a lingering reference file
924       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
925         "Found lingering reference file " + path);
926       if (!shouldFixReferenceFiles()) continue;
927 
928       // Now, trying to fix it since requested
929       boolean success = false;
930       String pathStr = path.toString();
931 
932       // A reference file path should be like
933       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
934       // Up 5 directories to get the root folder.
935       // So the file will be sidelined to a similar folder structure.
936       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
937       for (int i = 0; index > 0 && i < 5; i++) {
938         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
939       }
940       if (index > 0) {
941         Path rootDir = getSidelineDir();
942         Path dst = new Path(rootDir, pathStr.substring(index + 1));
943         fs.mkdirs(dst.getParent());
944         LOG.info("Trying to sildeline reference file "
945           + path + " to " + dst);
946         setShouldRerun();
947 
948         success = fs.rename(path, dst);
949       }
950       if (!success) {
951         LOG.error("Failed to sideline reference file " + path);
952       }
953     }
954   }
955 
956   /**
957    * TODO -- need to add tests for this.
958    */
959   private void reportEmptyMetaCells() {
960     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
961       emptyRegionInfoQualifiers.size());
962     if (details) {
963       for (Result r: emptyRegionInfoQualifiers) {
964         errors.print("  " + r);
965       }
966     }
967   }
968 
969   /**
970    * TODO -- need to add tests for this.
971    */
972   private void reportTablesInFlux() {
973     AtomicInteger numSkipped = new AtomicInteger(0);
974     HTableDescriptor[] allTables = getTables(numSkipped);
975     errors.print("Number of Tables: " + allTables.length);
976     if (details) {
977       if (numSkipped.get() > 0) {
978         errors.detail("Number of Tables in flux: " + numSkipped.get());
979       }
980       for (HTableDescriptor td : allTables) {
981         errors.detail("  Table: " + td.getTableName() + "\t" +
982                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
983                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
984                            " families: " + td.getFamilies().size());
985       }
986     }
987   }
988 
989   public ErrorReporter getErrors() {
990     return errors;
991   }
992 
993   /**
994    * Read the .regioninfo file from the file system.  If there is no
995    * .regioninfo, add it to the orphan hdfs region list.
996    */
997   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
998     Path regionDir = hbi.getHdfsRegionDir();
999     if (regionDir == null) {
1000       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1001       return;
1002     }
1003 
1004     if (hbi.hdfsEntry.hri != null) {
1005       // already loaded data
1006       return;
1007     }
1008 
1009     FileSystem fs = FileSystem.get(getConf());
1010     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1011     LOG.debug("HRegionInfo read: " + hri.toString());
1012     hbi.hdfsEntry.hri = hri;
1013   }
1014 
1015   /**
1016    * Exception thrown when a integrity repair operation fails in an
1017    * unresolvable way.
1018    */
1019   public static class RegionRepairException extends IOException {
1020     private static final long serialVersionUID = 1L;
1021     final IOException ioe;
1022     public RegionRepairException(String s, IOException ioe) {
1023       super(s);
1024       this.ioe = ioe;
1025     }
1026   }
1027 
1028   /**
1029    * Populate hbi's from regionInfos loaded from file system.
1030    */
1031   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1032       throws IOException, InterruptedException {
1033     tablesInfo.clear(); // regenerating the data
1034     // generate region split structure
1035     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1036 
1037     // Parallelized read of .regioninfo files.
1038     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1039     List<Future<Void>> hbiFutures;
1040 
1041     for (HbckInfo hbi : hbckInfos) {
1042       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1043       hbis.add(work);
1044     }
1045 
1046     // Submit and wait for completion
1047     hbiFutures = executor.invokeAll(hbis);
1048 
1049     for(int i=0; i<hbiFutures.size(); i++) {
1050       WorkItemHdfsRegionInfo work = hbis.get(i);
1051       Future<Void> f = hbiFutures.get(i);
1052       try {
1053         f.get();
1054       } catch(ExecutionException e) {
1055         LOG.warn("Failed to read .regioninfo file for region " +
1056               work.hbi.getRegionNameAsString(), e.getCause());
1057       }
1058     }
1059 
1060     Path hbaseRoot = FSUtils.getRootDir(getConf());
1061     FileSystem fs = hbaseRoot.getFileSystem(getConf());
1062     // serialized table info gathering.
1063     for (HbckInfo hbi: hbckInfos) {
1064 
1065       if (hbi.getHdfsHRI() == null) {
1066         // was an orphan
1067         continue;
1068       }
1069 
1070 
1071       // get table name from hdfs, populate various HBaseFsck tables.
1072       TableName tableName = hbi.getTableName();
1073       if (tableName == null) {
1074         // There was an entry in hbase:meta not in the HDFS?
1075         LOG.warn("tableName was null for: " + hbi);
1076         continue;
1077       }
1078 
1079       TableInfo modTInfo = tablesInfo.get(tableName);
1080       if (modTInfo == null) {
1081         // only executed once per table.
1082         modTInfo = new TableInfo(tableName);
1083         tablesInfo.put(tableName, modTInfo);
1084         try {
1085           TableDescriptor htd =
1086               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1087           modTInfo.htds.add(htd.getHTableDescriptor());
1088         } catch (IOException ioe) {
1089           if (!orphanTableDirs.containsKey(tableName)) {
1090             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1091             //should only report once for each table
1092             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1093                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1094             Set<String> columns = new HashSet<String>();
1095             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1096           }
1097         }
1098       }
1099       if (!hbi.isSkipChecks()) {
1100         modTInfo.addRegionInfo(hbi);
1101       }
1102     }
1103 
1104     loadTableInfosForTablesWithNoRegion();
1105 
1106     return tablesInfo;
1107   }
1108 
1109   /**
1110    * To get the column family list according to the column family dirs
1111    * @param columns
1112    * @param hbi
1113    * @return a set of column families
1114    * @throws IOException
1115    */
1116   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1117     Path regionDir = hbi.getHdfsRegionDir();
1118     FileSystem fs = regionDir.getFileSystem(getConf());
1119     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1120     for (FileStatus subdir : subDirs) {
1121       String columnfamily = subdir.getPath().getName();
1122       columns.add(columnfamily);
1123     }
1124     return columns;
1125   }
1126 
1127   /**
1128    * To fabricate a .tableinfo file with following contents<br>
1129    * 1. the correct tablename <br>
1130    * 2. the correct colfamily list<br>
1131    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1132    * @throws IOException
1133    */
1134   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1135       Set<String> columns) throws IOException {
1136     if (columns ==null || columns.isEmpty()) return false;
1137     HTableDescriptor htd = new HTableDescriptor(tableName);
1138     for (String columnfamimly : columns) {
1139       htd.addFamily(new HColumnDescriptor(columnfamimly));
1140     }
1141     fstd.createTableDescriptor(new TableDescriptor(htd), true);
1142     return true;
1143   }
1144 
1145   /**
1146    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1147    * @throws IOException
1148    */
1149   public void fixEmptyMetaCells() throws IOException {
1150     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1151       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1152       for (Result region : emptyRegionInfoQualifiers) {
1153         deleteMetaRegion(region.getRow());
1154         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1155       }
1156       emptyRegionInfoQualifiers.clear();
1157     }
1158   }
1159 
1160   /**
1161    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1162    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1163    * 2. else create a default .tableinfo file with following items<br>
1164    * &nbsp;2.1 the correct tablename <br>
1165    * &nbsp;2.2 the correct colfamily list<br>
1166    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1167    * @throws IOException
1168    */
1169   public void fixOrphanTables() throws IOException {
1170     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1171 
1172       List<TableName> tmpList = new ArrayList<TableName>();
1173       tmpList.addAll(orphanTableDirs.keySet());
1174       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1175       Iterator<Entry<TableName, Set<String>>> iter =
1176           orphanTableDirs.entrySet().iterator();
1177       int j = 0;
1178       int numFailedCase = 0;
1179       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1180       while (iter.hasNext()) {
1181         Entry<TableName, Set<String>> entry =
1182             iter.next();
1183         TableName tableName = entry.getKey();
1184         LOG.info("Trying to fix orphan table error: " + tableName);
1185         if (j < htds.length) {
1186           if (tableName.equals(htds[j].getTableName())) {
1187             HTableDescriptor htd = htds[j];
1188             LOG.info("fixing orphan table: " + tableName + " from cache");
1189             fstd.createTableDescriptor(new TableDescriptor(htd), true);
1190             j++;
1191             iter.remove();
1192           }
1193         } else {
1194           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1195             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1196             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1197             iter.remove();
1198           } else {
1199             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1200             numFailedCase++;
1201           }
1202         }
1203         fixes++;
1204       }
1205 
1206       if (orphanTableDirs.isEmpty()) {
1207         // all orphanTableDirs are luckily recovered
1208         // re-run doFsck after recovering the .tableinfo file
1209         setShouldRerun();
1210         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1211       } else if (numFailedCase > 0) {
1212         LOG.error("Failed to fix " + numFailedCase
1213             + " OrphanTables with default .tableinfo files");
1214       }
1215 
1216     }
1217     //cleanup the list
1218     orphanTableDirs.clear();
1219 
1220   }
1221 
1222   /**
1223    * This borrows code from MasterFileSystem.bootstrap(). Explicitly creates it's own WAL, so be
1224    * sure to close it as well as the region when you're finished.
1225    *
1226    * @return an open hbase:meta HRegion
1227    */
1228   private HRegion createNewMeta() throws IOException {
1229     Path rootdir = FSUtils.getRootDir(getConf());
1230     Configuration c = getConf();
1231     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1232     HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1233     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1234     // The WAL subsystem will use the default rootDir rather than the passed in rootDir
1235     // unless I pass along via the conf.
1236     Configuration confForWAL = new Configuration(c);
1237     confForWAL.set(HConstants.HBASE_DIR, rootdir.toString());
1238     WAL wal = (new WALFactory(confForWAL,
1239         Collections.<WALActionsListener>singletonList(new MetricsWAL()),
1240         "hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))).
1241         getWAL(metaHRI.getEncodedNameAsBytes());
1242     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal);
1243     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1244     return meta;
1245   }
1246 
1247   /**
1248    * Generate set of puts to add to new meta.  This expects the tables to be
1249    * clean with no overlaps or holes.  If there are any problems it returns null.
1250    *
1251    * @return An array list of puts to do in bulk, null if tables have problems
1252    */
1253   private ArrayList<Put> generatePuts(
1254       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1255     ArrayList<Put> puts = new ArrayList<Put>();
1256     boolean hasProblems = false;
1257     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1258       TableName name = e.getKey();
1259 
1260       // skip "hbase:meta"
1261       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1262         continue;
1263       }
1264 
1265       TableInfo ti = e.getValue();
1266       puts.add(MetaTableAccessor
1267           .makePutFromTableState(new TableState(ti.tableName, TableState.State.ENABLED)));
1268       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1269           .entrySet()) {
1270         Collection<HbckInfo> his = spl.getValue();
1271         int sz = his.size();
1272         if (sz != 1) {
1273           // problem
1274           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1275               + " had " +  sz + " regions instead of exactly 1." );
1276           hasProblems = true;
1277           continue;
1278         }
1279 
1280         // add the row directly to meta.
1281         HbckInfo hi = his.iterator().next();
1282         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1283         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1284         puts.add(p);
1285       }
1286     }
1287     return hasProblems ? null : puts;
1288   }
1289 
1290   /**
1291    * Suggest fixes for each table
1292    */
1293   private void suggestFixes(
1294       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1295     for (TableInfo tInfo : tablesInfo.values()) {
1296       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1297       tInfo.checkRegionChain(handler);
1298     }
1299   }
1300 
1301   /**
1302    * Rebuilds meta from information in hdfs/fs.  Depends on configuration settings passed into
1303    * hbck constructor to point to a particular fs/dir. Assumes HBase is OFFLINE.
1304    *
1305    * @param fix flag that determines if method should attempt to fix holes
1306    * @return true if successful, false if attempt failed.
1307    */
1308   public boolean rebuildMeta(boolean fix) throws IOException,
1309       InterruptedException {
1310 
1311     // TODO check to make sure hbase is offline. (or at least the table
1312     // currently being worked on is off line)
1313 
1314     // Determine what's on HDFS
1315     LOG.info("Loading HBase regioninfo from HDFS...");
1316     loadHdfsRegionDirs(); // populating regioninfo table.
1317 
1318     int errs = errors.getErrorList().size();
1319     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1320     checkHdfsIntegrity(false, false);
1321 
1322     // make sure ok.
1323     if (errors.getErrorList().size() != errs) {
1324       // While in error state, iterate until no more fixes possible
1325       while(true) {
1326         fixes = 0;
1327         suggestFixes(tablesInfo);
1328         errors.clear();
1329         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1330         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1331 
1332         int errCount = errors.getErrorList().size();
1333 
1334         if (fixes == 0) {
1335           if (errCount > 0) {
1336             return false; // failed to fix problems.
1337           } else {
1338             break; // no fixes and no problems? drop out and fix stuff!
1339           }
1340         }
1341       }
1342     }
1343 
1344     // we can rebuild, move old meta out of the way and start
1345     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1346     Path backupDir = sidelineOldMeta();
1347 
1348     LOG.info("Creating new hbase:meta");
1349     HRegion meta = createNewMeta();
1350 
1351     // populate meta
1352     List<Put> puts = generatePuts(tablesInfo);
1353     if (puts == null) {
1354       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1355         "You may need to restore the previously sidelined hbase:meta");
1356       return false;
1357     }
1358     meta.batchMutate(puts.toArray(new Put[puts.size()]));
1359     meta.close();
1360     if (meta.getWAL() != null) {
1361       meta.getWAL().close();
1362     }
1363     LOG.info("Success! hbase:meta table rebuilt.");
1364     LOG.info("Old hbase:meta is moved into " + backupDir);
1365     return true;
1366   }
1367 
1368   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1369       boolean fixOverlaps) throws IOException {
1370     LOG.info("Checking HBase region split map from HDFS data...");
1371     for (TableInfo tInfo : tablesInfo.values()) {
1372       TableIntegrityErrorHandler handler;
1373       if (fixHoles || fixOverlaps) {
1374         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1375           fixHoles, fixOverlaps);
1376       } else {
1377         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1378       }
1379       if (!tInfo.checkRegionChain(handler)) {
1380         // should dump info as well.
1381         errors.report("Found inconsistency in table " + tInfo.getName());
1382       }
1383     }
1384     return tablesInfo;
1385   }
1386 
1387   private Path getSidelineDir() throws IOException {
1388     if (sidelineDir == null) {
1389       Path hbaseDir = FSUtils.getRootDir(getConf());
1390       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1391       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1392           + startMillis);
1393     }
1394     return sidelineDir;
1395   }
1396 
1397   /**
1398    * Sideline a region dir (instead of deleting it)
1399    */
1400   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1401     return sidelineRegionDir(fs, null, hi);
1402   }
1403 
1404   /**
1405    * Sideline a region dir (instead of deleting it)
1406    *
1407    * @param parentDir if specified, the region will be sidelined to
1408    * folder like .../parentDir/<table name>/<region name>. The purpose
1409    * is to group together similar regions sidelined, for example, those
1410    * regions should be bulk loaded back later on. If null, it is ignored.
1411    */
1412   Path sidelineRegionDir(FileSystem fs,
1413       String parentDir, HbckInfo hi) throws IOException {
1414     TableName tableName = hi.getTableName();
1415     Path regionDir = hi.getHdfsRegionDir();
1416 
1417     if (!fs.exists(regionDir)) {
1418       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1419       return null;
1420     }
1421 
1422     Path rootDir = getSidelineDir();
1423     if (parentDir != null) {
1424       rootDir = new Path(rootDir, parentDir);
1425     }
1426     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1427     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1428     fs.mkdirs(sidelineRegionDir);
1429     boolean success = false;
1430     FileStatus[] cfs =  fs.listStatus(regionDir);
1431     if (cfs == null) {
1432       LOG.info("Region dir is empty: " + regionDir);
1433     } else {
1434       for (FileStatus cf : cfs) {
1435         Path src = cf.getPath();
1436         Path dst =  new Path(sidelineRegionDir, src.getName());
1437         if (fs.isFile(src)) {
1438           // simple file
1439           success = fs.rename(src, dst);
1440           if (!success) {
1441             String msg = "Unable to rename file " + src +  " to " + dst;
1442             LOG.error(msg);
1443             throw new IOException(msg);
1444           }
1445           continue;
1446         }
1447 
1448         // is a directory.
1449         fs.mkdirs(dst);
1450 
1451         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1452         // FileSystem.rename is inconsistent with directories -- if the
1453         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1454         // it moves the src into the dst dir resulting in (foo/a/b).  If
1455         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1456         FileStatus[] hfiles = fs.listStatus(src);
1457         if (hfiles != null && hfiles.length > 0) {
1458           for (FileStatus hfile : hfiles) {
1459             success = fs.rename(hfile.getPath(), dst);
1460             if (!success) {
1461               String msg = "Unable to rename file " + src +  " to " + dst;
1462               LOG.error(msg);
1463               throw new IOException(msg);
1464             }
1465           }
1466         }
1467         LOG.debug("Sideline directory contents:");
1468         debugLsr(sidelineRegionDir);
1469       }
1470     }
1471 
1472     LOG.info("Removing old region dir: " + regionDir);
1473     success = fs.delete(regionDir, true);
1474     if (!success) {
1475       String msg = "Unable to delete dir " + regionDir;
1476       LOG.error(msg);
1477       throw new IOException(msg);
1478     }
1479     return sidelineRegionDir;
1480   }
1481 
1482   /**
1483    * Side line an entire table.
1484    */
1485   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1486       Path backupHbaseDir) throws IOException {
1487     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1488     if (fs.exists(tableDir)) {
1489       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1490       fs.mkdirs(backupTableDir.getParent());
1491       boolean success = fs.rename(tableDir, backupTableDir);
1492       if (!success) {
1493         throw new IOException("Failed to move  " + tableName + " from "
1494             +  tableDir + " to " + backupTableDir);
1495       }
1496     } else {
1497       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1498     }
1499   }
1500 
1501   /**
1502    * @return Path to backup of original directory
1503    */
1504   Path sidelineOldMeta() throws IOException {
1505     // put current hbase:meta aside.
1506     Path hbaseDir = FSUtils.getRootDir(getConf());
1507     FileSystem fs = hbaseDir.getFileSystem(getConf());
1508     Path backupDir = getSidelineDir();
1509     fs.mkdirs(backupDir);
1510 
1511     try {
1512       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1513     } catch (IOException e) {
1514         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1515             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1516             + hbaseDir.getName() + ".", e);
1517       throw e; // throw original exception
1518     }
1519     return backupDir;
1520   }
1521 
1522   /**
1523    * Load the list of disabled tables in ZK into local set.
1524    * @throws ZooKeeperConnectionException
1525    * @throws IOException
1526    */
1527   private void loadTableStates()
1528   throws IOException {
1529     tableStates = MetaTableAccessor.getTableStates(connection);
1530   }
1531 
1532   /**
1533    * Check if the specified region's table is disabled.
1534    * @param tableName table to check status of
1535    */
1536   private boolean isTableDisabled(TableName tableName) {
1537     return tableStates.containsKey(tableName)
1538         && tableStates.get(tableName)
1539         .inStates(TableState.State.DISABLED, TableState.State.DISABLING);
1540   }
1541 
1542   /**
1543    * Scan HDFS for all regions, recording their information into
1544    * regionInfoMap
1545    */
1546   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1547     Path rootDir = FSUtils.getRootDir(getConf());
1548     FileSystem fs = rootDir.getFileSystem(getConf());
1549 
1550     // list all tables from HDFS
1551     List<FileStatus> tableDirs = Lists.newArrayList();
1552 
1553     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1554 
1555     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1556     for (Path path : paths) {
1557       TableName tableName = FSUtils.getTableName(path);
1558        if ((!checkMetaOnly &&
1559            isTableIncluded(tableName)) ||
1560            tableName.equals(TableName.META_TABLE_NAME)) {
1561          tableDirs.add(fs.getFileStatus(path));
1562        }
1563     }
1564 
1565     // verify that version file exists
1566     if (!foundVersionFile) {
1567       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1568           "Version file does not exist in root dir " + rootDir);
1569       if (shouldFixVersionFile()) {
1570         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1571             + " file.");
1572         setShouldRerun();
1573         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1574             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1575             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1576             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1577       }
1578     }
1579 
1580     // level 1:  <HBASE_DIR>/*
1581     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1582     List<Future<Void>> dirsFutures;
1583 
1584     for (FileStatus tableDir : tableDirs) {
1585       LOG.debug("Loading region dirs from " +tableDir.getPath());
1586       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1587     }
1588 
1589     // Invoke and wait for Callables to complete
1590     dirsFutures = executor.invokeAll(dirs);
1591 
1592     for(Future<Void> f: dirsFutures) {
1593       try {
1594         f.get();
1595       } catch(ExecutionException e) {
1596         LOG.warn("Could not load region dir " , e.getCause());
1597       }
1598     }
1599   }
1600 
1601   /**
1602    * Record the location of the hbase:meta region as found in ZooKeeper.
1603    */
1604   private boolean recordMetaRegion() throws IOException {
1605     RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1606         HConstants.EMPTY_START_ROW, false, false);
1607     if (rl == null) {
1608       errors.reportError(ERROR_CODE.NULL_META_REGION,
1609           "META region was not found in Zookeeper");
1610       return false;
1611     }
1612     for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1613       // Check if Meta region is valid and existing
1614       if (metaLocation == null ) {
1615         errors.reportError(ERROR_CODE.NULL_META_REGION,
1616             "META region location is null");
1617         return false;
1618       }
1619       if (metaLocation.getRegionInfo() == null) {
1620         errors.reportError(ERROR_CODE.NULL_META_REGION,
1621             "META location regionInfo is null");
1622         return false;
1623       }
1624       if (metaLocation.getHostname() == null) {
1625         errors.reportError(ERROR_CODE.NULL_META_REGION,
1626             "META location hostName is null");
1627         return false;
1628       }
1629       ServerName sn = metaLocation.getServerName();
1630       MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis());
1631       HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1632       if (hbckInfo == null) {
1633         regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1634       } else {
1635         hbckInfo.metaEntry = m;
1636       }
1637     }
1638     return true;
1639   }
1640 
1641   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1642     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1643       @Override
1644       public void abort(String why, Throwable e) {
1645         LOG.error(why, e);
1646         System.exit(1);
1647       }
1648 
1649       @Override
1650       public boolean isAborted() {
1651         return false;
1652       }
1653 
1654     });
1655   }
1656 
1657   private ServerName getMetaRegionServerName(int replicaId)
1658   throws IOException, KeeperException {
1659     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1660     ServerName sn = null;
1661     try {
1662       sn = new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1663     } finally {
1664       zkw.close();
1665     }
1666     return sn;
1667   }
1668 
1669   /**
1670    * Contacts each regionserver and fetches metadata about regions.
1671    * @param regionServerList - the list of region servers to connect to
1672    * @throws IOException if a remote or network exception occurs
1673    */
1674   void processRegionServers(Collection<ServerName> regionServerList)
1675     throws IOException, InterruptedException {
1676 
1677     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1678     List<Future<Void>> workFutures;
1679 
1680     // loop to contact each region server in parallel
1681     for (ServerName rsinfo: regionServerList) {
1682       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1683     }
1684 
1685     workFutures = executor.invokeAll(workItems);
1686 
1687     for(int i=0; i<workFutures.size(); i++) {
1688       WorkItemRegion item = workItems.get(i);
1689       Future<Void> f = workFutures.get(i);
1690       try {
1691         f.get();
1692       } catch(ExecutionException e) {
1693         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1694             e.getCause());
1695       }
1696     }
1697   }
1698 
1699   /**
1700    * Check consistency of all regions that have been found in previous phases.
1701    */
1702   private void checkAndFixConsistency()
1703   throws IOException, KeeperException, InterruptedException {
1704     // Divide the checks in two phases. One for default/primary replicas and another
1705     // for the non-primary ones. Keeps code cleaner this way.
1706     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1707       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1708         checkRegionConsistency(e.getKey(), e.getValue());
1709       }
1710     }
1711     boolean prevHdfsCheck = shouldCheckHdfs();
1712     setCheckHdfs(false); //replicas don't have any hdfs data
1713     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1714     // deployed/undeployed replicas.
1715     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1716       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1717         checkRegionConsistency(e.getKey(), e.getValue());
1718       }
1719     }
1720     setCheckHdfs(prevHdfsCheck);
1721 
1722     if (shouldCheckHdfs()) {
1723       checkAndFixTableStates();
1724     }
1725   }
1726 
1727   /**
1728    * Check and fix table states, assumes full info available:
1729    * - tableInfos
1730    * - empty tables loaded
1731    */
1732   private void checkAndFixTableStates() throws IOException {
1733     // first check dangling states
1734     for (Entry<TableName, TableState> entry : tableStates.entrySet()) {
1735       TableName tableName = entry.getKey();
1736       TableState tableState = entry.getValue();
1737       TableInfo tableInfo = tablesInfo.get(tableName);
1738       if (isTableIncluded(tableName)
1739           && !tableName.isSystemTable()
1740           && tableInfo == null) {
1741         if (fixMeta) {
1742           MetaTableAccessor.deleteTableState(connection, tableName);
1743           TableState state = MetaTableAccessor.getTableState(connection, tableName);
1744           if (state != null) {
1745             errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1746                 tableName + " unable to delete dangling table state " + tableState);
1747           }
1748         } else {
1749           errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1750               tableName + " has dangling table state " + tableState);
1751         }
1752       }
1753     }
1754     // check that all tables have states
1755     for (TableName tableName : tablesInfo.keySet()) {
1756       if (isTableIncluded(tableName) && !tableStates.containsKey(tableName)) {
1757         if (fixMeta) {
1758           MetaTableAccessor.updateTableState(connection, tableName, TableState.State.ENABLED);
1759           TableState newState = MetaTableAccessor.getTableState(connection, tableName);
1760           if (newState == null) {
1761             errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1762                 "Unable to change state for table " + tableName + " in meta ");
1763           }
1764         } else {
1765           errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1766               tableName + " has no state in meta ");
1767         }
1768       }
1769     }
1770   }
1771 
1772   private void preCheckPermission() throws IOException, AccessDeniedException {
1773     if (shouldIgnorePreCheckPermission()) {
1774       return;
1775     }
1776 
1777     Path hbaseDir = FSUtils.getRootDir(getConf());
1778     FileSystem fs = hbaseDir.getFileSystem(getConf());
1779     UserProvider userProvider = UserProvider.instantiate(getConf());
1780     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1781     FileStatus[] files = fs.listStatus(hbaseDir);
1782     for (FileStatus file : files) {
1783       try {
1784         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1785       } catch (AccessDeniedException ace) {
1786         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1787         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1788           + " does not have write perms to " + file.getPath()
1789           + ". Please rerun hbck as hdfs user " + file.getOwner());
1790         throw ace;
1791       }
1792     }
1793   }
1794 
1795   /**
1796    * Deletes region from meta table
1797    */
1798   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1799     deleteMetaRegion(hi.metaEntry.getRegionName());
1800   }
1801 
1802   /**
1803    * Deletes region from meta table
1804    */
1805   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1806     Delete d = new Delete(metaKey);
1807     meta.delete(d);
1808     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1809   }
1810 
1811   /**
1812    * Reset the split parent region info in meta table
1813    */
1814   private void resetSplitParent(HbckInfo hi) throws IOException {
1815     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1816     Delete d = new Delete(hi.metaEntry.getRegionName());
1817     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1818     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1819     mutations.add(d);
1820 
1821     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1822     hri.setOffline(false);
1823     hri.setSplit(false);
1824     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1825     mutations.add(p);
1826 
1827     meta.mutateRow(mutations);
1828     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1829   }
1830 
1831   /**
1832    * This backwards-compatibility wrapper for permanently offlining a region
1833    * that should not be alive.  If the region server does not support the
1834    * "offline" method, it will use the closest unassign method instead.  This
1835    * will basically work until one attempts to disable or delete the affected
1836    * table.  The problem has to do with in-memory only master state, so
1837    * restarting the HMaster or failing over to another should fix this.
1838    */
1839   private void offline(byte[] regionName) throws IOException {
1840     String regionString = Bytes.toStringBinary(regionName);
1841     if (!rsSupportsOffline) {
1842       LOG.warn("Using unassign region " + regionString
1843           + " instead of using offline method, you should"
1844           + " restart HMaster after these repairs");
1845       admin.unassign(regionName, true);
1846       return;
1847     }
1848 
1849     // first time we assume the rs's supports #offline.
1850     try {
1851       LOG.info("Offlining region " + regionString);
1852       admin.offline(regionName);
1853     } catch (IOException ioe) {
1854       String notFoundMsg = "java.lang.NoSuchMethodException: " +
1855         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
1856       if (ioe.getMessage().contains(notFoundMsg)) {
1857         LOG.warn("Using unassign region " + regionString
1858             + " instead of using offline method, you should"
1859             + " restart HMaster after these repairs");
1860         rsSupportsOffline = false; // in the future just use unassign
1861         admin.unassign(regionName, true);
1862         return;
1863       }
1864       throw ioe;
1865     }
1866   }
1867 
1868   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
1869     undeployRegionsForHbi(hi);
1870     // undeploy replicas of the region (but only if the method is invoked for the primary)
1871     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1872       return;
1873     }
1874     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1875     for (int i = 1; i < numReplicas; i++) {
1876       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
1877       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
1878           hi.getPrimaryHRIForDeployedReplica(), i);
1879       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1880       if (h != null) {
1881         undeployRegionsForHbi(h);
1882         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
1883         //in consistency checks
1884         h.setSkipChecks(true);
1885       }
1886     }
1887   }
1888 
1889   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
1890     for (OnlineEntry rse : hi.deployedEntries) {
1891       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
1892       try {
1893         HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
1894         offline(rse.hri.getRegionName());
1895       } catch (IOException ioe) {
1896         LOG.warn("Got exception when attempting to offline region "
1897             + Bytes.toString(rse.hri.getRegionName()), ioe);
1898       }
1899     }
1900   }
1901 
1902   /**
1903    * Attempts to undeploy a region from a region server based in information in
1904    * META.  Any operations that modify the file system should make sure that
1905    * its corresponding region is not deployed to prevent data races.
1906    *
1907    * A separate call is required to update the master in-memory region state
1908    * kept in the AssignementManager.  Because disable uses this state instead of
1909    * that found in META, we can't seem to cleanly disable/delete tables that
1910    * have been hbck fixed.  When used on a version of HBase that does not have
1911    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
1912    * restart or failover may be required.
1913    */
1914   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
1915     if (hi.metaEntry == null && hi.hdfsEntry == null) {
1916       undeployRegions(hi);
1917       return;
1918     }
1919 
1920     // get assignment info and hregioninfo from meta.
1921     Get get = new Get(hi.getRegionName());
1922     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1923     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
1924     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
1925     // also get the locations of the replicas to close if the primary region is being closed
1926     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1927       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1928       for (int i = 0; i < numReplicas; i++) {
1929         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
1930         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
1931       }
1932     }
1933     Result r = meta.get(get);
1934     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
1935     if (rl == null) {
1936       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
1937           " since meta does not have handle to reach it");
1938       return;
1939     }
1940     for (HRegionLocation h : rl.getRegionLocations()) {
1941       ServerName serverName = h.getServerName();
1942       if (serverName == null) {
1943         errors.reportError("Unable to close region "
1944             + hi.getRegionNameAsString() +  " because meta does not "
1945             + "have handle to reach it.");
1946         continue;
1947       }
1948       HRegionInfo hri = h.getRegionInfo();
1949       if (hri == null) {
1950         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
1951             + " because hbase:meta had invalid or missing "
1952             + HConstants.CATALOG_FAMILY_STR + ":"
1953             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
1954             + " qualifier value.");
1955         continue;
1956       }
1957       // close the region -- close files and remove assignment
1958       HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
1959     }
1960   }
1961 
1962   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
1963     KeeperException, InterruptedException {
1964     // If we are trying to fix the errors
1965     if (shouldFixAssignments()) {
1966       errors.print(msg);
1967       undeployRegions(hbi);
1968       setShouldRerun();
1969       HRegionInfo hri = hbi.getHdfsHRI();
1970       if (hri == null) {
1971         hri = hbi.metaEntry;
1972       }
1973       HBaseFsckRepair.fixUnassigned(admin, hri);
1974       HBaseFsckRepair.waitUntilAssigned(admin, hri);
1975 
1976       // also assign replicas if needed (do it only when this call operates on a primary replica)
1977       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
1978       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
1979       for (int i = 1; i < replicationCount; i++) {
1980         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
1981         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1982         if (h != null) {
1983           undeployRegions(h);
1984           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
1985           //in consistency checks
1986           h.setSkipChecks(true);
1987         }
1988         HBaseFsckRepair.fixUnassigned(admin, hri);
1989         HBaseFsckRepair.waitUntilAssigned(admin, hri);
1990       }
1991 
1992     }
1993   }
1994 
1995   /**
1996    * Check a single region for consistency and correct deployment.
1997    */
1998   private void checkRegionConsistency(final String key, final HbckInfo hbi)
1999   throws IOException, KeeperException, InterruptedException {
2000 
2001     if (hbi.isSkipChecks()) return;
2002     String descriptiveName = hbi.toString();
2003     boolean inMeta = hbi.metaEntry != null;
2004     // In case not checking HDFS, assume the region is on HDFS
2005     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2006     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2007     boolean isDeployed = !hbi.deployedOn.isEmpty();
2008     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2009     boolean deploymentMatchesMeta =
2010       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2011       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2012     boolean splitParent =
2013         inMeta && hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2014     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry.getTable());
2015     boolean recentlyModified = inHdfs &&
2016       hbi.getModTime() + timelag > System.currentTimeMillis();
2017 
2018     // ========== First the healthy cases =============
2019     if (hbi.containsOnlyHdfsEdits()) {
2020       return;
2021     }
2022     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2023       return;
2024     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2025       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2026         "tabled that is not deployed");
2027       return;
2028     } else if (recentlyModified) {
2029       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2030       return;
2031     }
2032     // ========== Cases where the region is not in hbase:meta =============
2033     else if (!inMeta && !inHdfs && !isDeployed) {
2034       // We shouldn't have record of this region at all then!
2035       assert false : "Entry for region with no data";
2036     } else if (!inMeta && !inHdfs && isDeployed) {
2037       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2038           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2039           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2040       if (shouldFixAssignments()) {
2041         undeployRegions(hbi);
2042       }
2043 
2044     } else if (!inMeta && inHdfs && !isDeployed) {
2045       if (hbi.isMerged()) {
2046         // This region has already been merged, the remaining hdfs file will be
2047         // cleaned by CatalogJanitor later
2048         hbi.setSkipChecks(true);
2049         LOG.info("Region " + descriptiveName
2050             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2051         return;
2052       }
2053       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2054           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2055           "or deployed on any region server");
2056       // restore region consistency of an adopted orphan
2057       if (shouldFixMeta()) {
2058         if (!hbi.isHdfsRegioninfoPresent()) {
2059           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2060               +  " in table integrity repair phase if -fixHdfsOrphans was" +
2061               " used.");
2062           return;
2063         }
2064 
2065         HRegionInfo hri = hbi.getHdfsHRI();
2066         TableInfo tableInfo = tablesInfo.get(hri.getTable());
2067         if (tableInfo.regionsFromMeta.isEmpty()) {
2068           for (HbckInfo h : regionInfoMap.values()) {
2069             if (hri.getTable().equals(h.getTableName())) {
2070               if (h.metaEntry != null) tableInfo.regionsFromMeta
2071                   .add((HRegionInfo) h.metaEntry);
2072             }
2073           }
2074           Collections.sort(tableInfo.regionsFromMeta);
2075         }
2076         for (HRegionInfo region : tableInfo.regionsFromMeta) {
2077           if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2078               && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2079                 hri.getEndKey()) >= 0)
2080               && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2081             if(region.isSplit() || region.isOffline()) continue;
2082             Path regionDir = hbi.getHdfsRegionDir();
2083             FileSystem fs = regionDir.getFileSystem(getConf());
2084             List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2085             for (Path familyDir : familyDirs) {
2086               List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2087               for (Path referenceFilePath : referenceFilePaths) {
2088                 Path parentRegionDir =
2089                     StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2090                 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2091                   LOG.warn(hri + " start and stop keys are in the range of " + region
2092                       + ". The region might not be cleaned up from hdfs when region " + region
2093                       + " split failed. Hence deleting from hdfs.");
2094                   HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2095                     regionDir.getParent(), hri);
2096                   return;
2097                 }
2098               }
2099             }
2100           }
2101         }
2102         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2103         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2104         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2105             admin.getClusterStatus().getServers(), numReplicas);
2106 
2107         tryAssignmentRepair(hbi, "Trying to reassign region...");
2108       }
2109 
2110     } else if (!inMeta && inHdfs && isDeployed) {
2111       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2112           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2113       debugLsr(hbi.getHdfsRegionDir());
2114       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2115         // for replicas, this means that we should undeploy the region (we would have
2116         // gone over the primaries and fixed meta holes in first phase under
2117         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
2118         // this stage unless unwanted replica)
2119         if (shouldFixAssignments()) {
2120           undeployRegionsForHbi(hbi);
2121         }
2122       }
2123       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2124         if (!hbi.isHdfsRegioninfoPresent()) {
2125           LOG.error("This should have been repaired in table integrity repair phase");
2126           return;
2127         }
2128 
2129         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2130         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2131         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2132             admin.getClusterStatus().getServers(), numReplicas);
2133         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2134       }
2135 
2136     // ========== Cases where the region is in hbase:meta =============
2137     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2138       // check whether this is an actual error, or just transient state where parent
2139       // is not cleaned
2140       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2141         // check that split daughters are there
2142         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2143         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2144         if (infoA != null && infoB != null) {
2145           // we already processed or will process daughters. Move on, nothing to see here.
2146           hbi.setSkipChecks(true);
2147           return;
2148         }
2149       }
2150       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2151           + descriptiveName + " is a split parent in META, in HDFS, "
2152           + "and not deployed on any region server. This could be transient.");
2153       if (shouldFixSplitParents()) {
2154         setShouldRerun();
2155         resetSplitParent(hbi);
2156       }
2157     } else if (inMeta && !inHdfs && !isDeployed) {
2158       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2159           + descriptiveName + " found in META, but not in HDFS "
2160           + "or deployed on any region server.");
2161       if (shouldFixMeta()) {
2162         deleteMetaRegion(hbi);
2163       }
2164     } else if (inMeta && !inHdfs && isDeployed) {
2165       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2166           + " found in META, but not in HDFS, " +
2167           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2168       // We treat HDFS as ground truth.  Any information in meta is transient
2169       // and equivalent data can be regenerated.  So, lets unassign and remove
2170       // these problems from META.
2171       if (shouldFixAssignments()) {
2172         errors.print("Trying to fix unassigned region...");
2173         undeployRegions(hbi);
2174       }
2175       if (shouldFixMeta()) {
2176         // wait for it to complete
2177         deleteMetaRegion(hbi);
2178       }
2179     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2180       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2181           + " not deployed on any region server.");
2182       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2183     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2184       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2185           "Region " + descriptiveName + " should not be deployed according " +
2186           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2187       if (shouldFixAssignments()) {
2188         errors.print("Trying to close the region " + descriptiveName);
2189         setShouldRerun();
2190         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2191       }
2192     } else if (inMeta && inHdfs && isMultiplyDeployed) {
2193       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2194           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2195           + " but is multiply assigned to region servers " +
2196           Joiner.on(", ").join(hbi.deployedOn));
2197       // If we are trying to fix the errors
2198       if (shouldFixAssignments()) {
2199         errors.print("Trying to fix assignment error...");
2200         setShouldRerun();
2201         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2202       }
2203     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2204       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2205           + descriptiveName + " listed in hbase:meta on region server " +
2206           hbi.metaEntry.regionServer + " but found on region server " +
2207           hbi.deployedOn.get(0));
2208       // If we are trying to fix the errors
2209       if (shouldFixAssignments()) {
2210         errors.print("Trying to fix assignment error...");
2211         setShouldRerun();
2212         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2213         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2214       }
2215     } else {
2216       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2217           " is in an unforeseen state:" +
2218           " inMeta=" + inMeta +
2219           " inHdfs=" + inHdfs +
2220           " isDeployed=" + isDeployed +
2221           " isMultiplyDeployed=" + isMultiplyDeployed +
2222           " deploymentMatchesMeta=" + deploymentMatchesMeta +
2223           " shouldBeDeployed=" + shouldBeDeployed);
2224     }
2225   }
2226 
2227   /**
2228    * Checks tables integrity. Goes over all regions and scans the tables.
2229    * Collects all the pieces for each table and checks if there are missing,
2230    * repeated or overlapping ones.
2231    * @throws IOException
2232    */
2233   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2234     tablesInfo = new TreeMap<TableName,TableInfo> ();
2235     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2236     for (HbckInfo hbi : regionInfoMap.values()) {
2237       // Check only valid, working regions
2238       if (hbi.metaEntry == null) {
2239         // this assumes that consistency check has run loadMetaEntry
2240         Path p = hbi.getHdfsRegionDir();
2241         if (p == null) {
2242           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2243         }
2244 
2245         // TODO test.
2246         continue;
2247       }
2248       if (hbi.metaEntry.regionServer == null) {
2249         errors.detail("Skipping region because no region server: " + hbi);
2250         continue;
2251       }
2252       if (hbi.metaEntry.isOffline()) {
2253         errors.detail("Skipping region because it is offline: " + hbi);
2254         continue;
2255       }
2256       if (hbi.containsOnlyHdfsEdits()) {
2257         errors.detail("Skipping region because it only contains edits" + hbi);
2258         continue;
2259       }
2260 
2261       // Missing regionDir or over-deployment is checked elsewhere. Include
2262       // these cases in modTInfo, so we can evaluate those regions as part of
2263       // the region chain in META
2264       //if (hbi.foundRegionDir == null) continue;
2265       //if (hbi.deployedOn.size() != 1) continue;
2266       if (hbi.deployedOn.size() == 0) continue;
2267 
2268       // We should be safe here
2269       TableName tableName = hbi.metaEntry.getTable();
2270       TableInfo modTInfo = tablesInfo.get(tableName);
2271       if (modTInfo == null) {
2272         modTInfo = new TableInfo(tableName);
2273       }
2274       for (ServerName server : hbi.deployedOn) {
2275         modTInfo.addServer(server);
2276       }
2277 
2278       if (!hbi.isSkipChecks()) {
2279         modTInfo.addRegionInfo(hbi);
2280       }
2281 
2282       tablesInfo.put(tableName, modTInfo);
2283     }
2284 
2285     loadTableInfosForTablesWithNoRegion();
2286 
2287     for (TableInfo tInfo : tablesInfo.values()) {
2288       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2289       if (!tInfo.checkRegionChain(handler)) {
2290         errors.report("Found inconsistency in table " + tInfo.getName());
2291       }
2292     }
2293     return tablesInfo;
2294   }
2295 
2296   /** Loads table info's for tables that may not have been included, since there are no
2297    * regions reported for the table, but table dir is there in hdfs
2298    */
2299   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2300     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2301     for (HTableDescriptor htd : allTables.values()) {
2302       if (checkMetaOnly && !htd.isMetaTable()) {
2303         continue;
2304       }
2305 
2306       TableName tableName = htd.getTableName();
2307       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2308         TableInfo tableInfo = new TableInfo(tableName);
2309         tableInfo.htds.add(htd);
2310         tablesInfo.put(htd.getTableName(), tableInfo);
2311       }
2312     }
2313   }
2314 
2315   /**
2316    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2317    * @return number of file move fixes done to merge regions.
2318    */
2319   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2320     int fileMoves = 0;
2321     String thread = Thread.currentThread().getName();
2322     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2323     debugLsr(contained.getHdfsRegionDir());
2324 
2325     // rename the contained into the container.
2326     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2327     FileStatus[] dirs = null;
2328     try {
2329       dirs = fs.listStatus(contained.getHdfsRegionDir());
2330     } catch (FileNotFoundException fnfe) {
2331       // region we are attempting to merge in is not present!  Since this is a merge, there is
2332       // no harm skipping this region if it does not exist.
2333       if (!fs.exists(contained.getHdfsRegionDir())) {
2334         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2335             + " is missing. Assuming already sidelined or moved.");
2336       } else {
2337         sidelineRegionDir(fs, contained);
2338       }
2339       return fileMoves;
2340     }
2341 
2342     if (dirs == null) {
2343       if (!fs.exists(contained.getHdfsRegionDir())) {
2344         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2345             + " already sidelined.");
2346       } else {
2347         sidelineRegionDir(fs, contained);
2348       }
2349       return fileMoves;
2350     }
2351 
2352     for (FileStatus cf : dirs) {
2353       Path src = cf.getPath();
2354       Path dst =  new Path(targetRegionDir, src.getName());
2355 
2356       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2357         // do not copy the old .regioninfo file.
2358         continue;
2359       }
2360 
2361       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2362         // do not copy the .oldlogs files
2363         continue;
2364       }
2365 
2366       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2367       // FileSystem.rename is inconsistent with directories -- if the
2368       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2369       // it moves the src into the dst dir resulting in (foo/a/b).  If
2370       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2371       for (FileStatus hfile : fs.listStatus(src)) {
2372         boolean success = fs.rename(hfile.getPath(), dst);
2373         if (success) {
2374           fileMoves++;
2375         }
2376       }
2377       LOG.debug("[" + thread + "] Sideline directory contents:");
2378       debugLsr(targetRegionDir);
2379     }
2380 
2381     // if all success.
2382     sidelineRegionDir(fs, contained);
2383     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2384         getSidelineDir());
2385     debugLsr(contained.getHdfsRegionDir());
2386 
2387     return fileMoves;
2388   }
2389 
2390 
2391   static class WorkItemOverlapMerge implements Callable<Void> {
2392     private TableIntegrityErrorHandler handler;
2393     Collection<HbckInfo> overlapgroup;
2394 
2395     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2396       this.handler = handler;
2397       this.overlapgroup = overlapgroup;
2398     }
2399 
2400     @Override
2401     public Void call() throws Exception {
2402       handler.handleOverlapGroup(overlapgroup);
2403       return null;
2404     }
2405   };
2406 
2407 
2408   /**
2409    * Maintain information about a particular table.
2410    */
2411   public class TableInfo {
2412     TableName tableName;
2413     TreeSet <ServerName> deployedOn;
2414 
2415     // backwards regions
2416     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2417 
2418     // sidelined big overlapped regions
2419     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2420 
2421     // region split calculator
2422     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2423 
2424     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2425     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2426 
2427     // key = start split, values = set of splits in problem group
2428     final Multimap<byte[], HbckInfo> overlapGroups =
2429       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2430 
2431     // list of regions derived from meta entries.
2432     final List<HRegionInfo> regionsFromMeta = new ArrayList<HRegionInfo>();
2433 
2434     TableInfo(TableName name) {
2435       this.tableName = name;
2436       deployedOn = new TreeSet <ServerName>();
2437     }
2438 
2439     /**
2440      * @return descriptor common to all regions.  null if are none or multiple!
2441      */
2442     private HTableDescriptor getHTD() {
2443       if (htds.size() == 1) {
2444         return (HTableDescriptor)htds.toArray()[0];
2445       } else {
2446         LOG.error("None/Multiple table descriptors found for table '"
2447           + tableName + "' regions: " + htds);
2448       }
2449       return null;
2450     }
2451 
2452     public void addRegionInfo(HbckInfo hir) {
2453       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2454         // end key is absolute end key, just add it.
2455         // ignore replicas other than primary for these checks
2456         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2457         return;
2458       }
2459 
2460       // if not the absolute end key, check for cycle
2461       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2462         errors.reportError(
2463             ERROR_CODE.REGION_CYCLE,
2464             String.format("The endkey for this region comes before the "
2465                 + "startkey, startkey=%s, endkey=%s",
2466                 Bytes.toStringBinary(hir.getStartKey()),
2467                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2468         backwards.add(hir);
2469         return;
2470       }
2471 
2472       // main case, add to split calculator
2473       // ignore replicas other than primary for these checks
2474       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2475     }
2476 
2477     public void addServer(ServerName server) {
2478       this.deployedOn.add(server);
2479     }
2480 
2481     public TableName getName() {
2482       return tableName;
2483     }
2484 
2485     public int getNumRegions() {
2486       return sc.getStarts().size() + backwards.size();
2487     }
2488 
2489     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2490       ErrorReporter errors;
2491 
2492       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2493         this.errors = errors;
2494         setTableInfo(ti);
2495       }
2496 
2497       @Override
2498       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2499         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2500             "First region should start with an empty key.  You need to "
2501             + " create a new region and regioninfo in HDFS to plug the hole.",
2502             getTableInfo(), hi);
2503       }
2504 
2505       @Override
2506       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2507         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2508             "Last region should end with an empty key. You need to "
2509                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2510       }
2511 
2512       @Override
2513       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2514         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2515             "Region has the same start and end key.", getTableInfo(), hi);
2516       }
2517 
2518       @Override
2519       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2520         byte[] key = r1.getStartKey();
2521         // dup start key
2522         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2523             "Multiple regions have the same startkey: "
2524             + Bytes.toStringBinary(key), getTableInfo(), r1);
2525         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2526             "Multiple regions have the same startkey: "
2527             + Bytes.toStringBinary(key), getTableInfo(), r2);
2528       }
2529 
2530       @Override
2531       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2532         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2533             "There is an overlap in the region chain.",
2534             getTableInfo(), hi1, hi2);
2535       }
2536 
2537       @Override
2538       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2539         errors.reportError(
2540             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2541             "There is a hole in the region chain between "
2542                 + Bytes.toStringBinary(holeStart) + " and "
2543                 + Bytes.toStringBinary(holeStop)
2544                 + ".  You need to create a new .regioninfo and region "
2545                 + "dir in hdfs to plug the hole.");
2546       }
2547     };
2548 
2549     /**
2550      * This handler fixes integrity errors from hdfs information.  There are
2551      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2552      * 3) invalid regions.
2553      *
2554      * This class overrides methods that fix holes and the overlap group case.
2555      * Individual cases of particular overlaps are handled by the general
2556      * overlap group merge repair case.
2557      *
2558      * If hbase is online, this forces regions offline before doing merge
2559      * operations.
2560      */
2561     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2562       Configuration conf;
2563 
2564       boolean fixOverlaps = true;
2565 
2566       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2567           boolean fixHoles, boolean fixOverlaps) {
2568         super(ti, errors);
2569         this.conf = conf;
2570         this.fixOverlaps = fixOverlaps;
2571         // TODO properly use fixHoles
2572       }
2573 
2574       /**
2575        * This is a special case hole -- when the first region of a table is
2576        * missing from META, HBase doesn't acknowledge the existance of the
2577        * table.
2578        */
2579       @Override
2580       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2581         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2582             "First region should start with an empty key.  Creating a new " +
2583             "region and regioninfo in HDFS to plug the hole.",
2584             getTableInfo(), next);
2585         HTableDescriptor htd = getTableInfo().getHTD();
2586         // from special EMPTY_START_ROW to next region's startKey
2587         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2588             HConstants.EMPTY_START_ROW, next.getStartKey());
2589 
2590         // TODO test
2591         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2592         LOG.info("Table region start key was not empty.  Created new empty region: "
2593             + newRegion + " " +region);
2594         fixes++;
2595       }
2596 
2597       @Override
2598       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2599         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2600             "Last region should end with an empty key.  Creating a new "
2601                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2602         HTableDescriptor htd = getTableInfo().getHTD();
2603         // from curEndKey to EMPTY_START_ROW
2604         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2605             HConstants.EMPTY_START_ROW);
2606 
2607         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2608         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2609             + " " + region);
2610         fixes++;
2611       }
2612 
2613       /**
2614        * There is a hole in the hdfs regions that violates the table integrity
2615        * rules.  Create a new empty region that patches the hole.
2616        */
2617       @Override
2618       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2619         errors.reportError(
2620             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2621             "There is a hole in the region chain between "
2622                 + Bytes.toStringBinary(holeStartKey) + " and "
2623                 + Bytes.toStringBinary(holeStopKey)
2624                 + ".  Creating a new regioninfo and region "
2625                 + "dir in hdfs to plug the hole.");
2626         HTableDescriptor htd = getTableInfo().getHTD();
2627         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2628         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2629         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2630         fixes++;
2631       }
2632 
2633       /**
2634        * This takes set of overlapping regions and merges them into a single
2635        * region.  This covers cases like degenerate regions, shared start key,
2636        * general overlaps, duplicate ranges, and partial overlapping regions.
2637        *
2638        * Cases:
2639        * - Clean regions that overlap
2640        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2641        *
2642        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2643        */
2644       @Override
2645       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2646           throws IOException {
2647         Preconditions.checkNotNull(overlap);
2648         Preconditions.checkArgument(overlap.size() >0);
2649 
2650         if (!this.fixOverlaps) {
2651           LOG.warn("Not attempting to repair overlaps.");
2652           return;
2653         }
2654 
2655         if (overlap.size() > maxMerge) {
2656           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2657             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2658           if (sidelineBigOverlaps) {
2659             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2660             sidelineBigOverlaps(overlap);
2661           }
2662           return;
2663         }
2664 
2665         mergeOverlaps(overlap);
2666       }
2667 
2668       void mergeOverlaps(Collection<HbckInfo> overlap)
2669           throws IOException {
2670         String thread = Thread.currentThread().getName();
2671         LOG.info("== [" + thread + "] Merging regions into one region: "
2672           + Joiner.on(",").join(overlap));
2673         // get the min / max range and close all concerned regions
2674         Pair<byte[], byte[]> range = null;
2675         for (HbckInfo hi : overlap) {
2676           if (range == null) {
2677             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2678           } else {
2679             if (RegionSplitCalculator.BYTES_COMPARATOR
2680                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2681               range.setFirst(hi.getStartKey());
2682             }
2683             if (RegionSplitCalculator.BYTES_COMPARATOR
2684                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2685               range.setSecond(hi.getEndKey());
2686             }
2687           }
2688           // need to close files so delete can happen.
2689           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2690           LOG.debug("[" + thread + "] Contained region dir before close");
2691           debugLsr(hi.getHdfsRegionDir());
2692           try {
2693             LOG.info("[" + thread + "] Closing region: " + hi);
2694             closeRegion(hi);
2695           } catch (IOException ioe) {
2696             LOG.warn("[" + thread + "] Was unable to close region " + hi
2697               + ".  Just continuing... ", ioe);
2698           } catch (InterruptedException e) {
2699             LOG.warn("[" + thread + "] Was unable to close region " + hi
2700               + ".  Just continuing... ", e);
2701           }
2702 
2703           try {
2704             LOG.info("[" + thread + "] Offlining region: " + hi);
2705             offline(hi.getRegionName());
2706           } catch (IOException ioe) {
2707             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2708               + ".  Just continuing... ", ioe);
2709           }
2710         }
2711 
2712         // create new empty container region.
2713         HTableDescriptor htd = getTableInfo().getHTD();
2714         // from start key to end Key
2715         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2716             range.getSecond());
2717         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2718         LOG.info("[" + thread + "] Created new empty container region: " +
2719             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2720         debugLsr(region.getRegionFileSystem().getRegionDir());
2721 
2722         // all target regions are closed, should be able to safely cleanup.
2723         boolean didFix= false;
2724         Path target = region.getRegionFileSystem().getRegionDir();
2725         for (HbckInfo contained : overlap) {
2726           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2727           int merges = mergeRegionDirs(target, contained);
2728           if (merges > 0) {
2729             didFix = true;
2730           }
2731         }
2732         if (didFix) {
2733           fixes++;
2734         }
2735       }
2736 
2737       /**
2738        * Sideline some regions in a big overlap group so that it
2739        * will have fewer regions, and it is easier to merge them later on.
2740        *
2741        * @param bigOverlap the overlapped group with regions more than maxMerge
2742        * @throws IOException
2743        */
2744       void sidelineBigOverlaps(
2745           Collection<HbckInfo> bigOverlap) throws IOException {
2746         int overlapsToSideline = bigOverlap.size() - maxMerge;
2747         if (overlapsToSideline > maxOverlapsToSideline) {
2748           overlapsToSideline = maxOverlapsToSideline;
2749         }
2750         List<HbckInfo> regionsToSideline =
2751           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2752         FileSystem fs = FileSystem.get(conf);
2753         for (HbckInfo regionToSideline: regionsToSideline) {
2754           try {
2755             LOG.info("Closing region: " + regionToSideline);
2756             closeRegion(regionToSideline);
2757           } catch (IOException ioe) {
2758             LOG.warn("Was unable to close region " + regionToSideline
2759               + ".  Just continuing... ", ioe);
2760           } catch (InterruptedException e) {
2761             LOG.warn("Was unable to close region " + regionToSideline
2762               + ".  Just continuing... ", e);
2763           }
2764 
2765           try {
2766             LOG.info("Offlining region: " + regionToSideline);
2767             offline(regionToSideline.getRegionName());
2768           } catch (IOException ioe) {
2769             LOG.warn("Unable to offline region from master: " + regionToSideline
2770               + ".  Just continuing... ", ioe);
2771           }
2772 
2773           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2774           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2775           if (sidelineRegionDir != null) {
2776             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2777             LOG.info("After sidelined big overlapped region: "
2778               + regionToSideline.getRegionNameAsString()
2779               + " to " + sidelineRegionDir.toString());
2780             fixes++;
2781           }
2782         }
2783       }
2784     }
2785 
2786     /**
2787      * Check the region chain (from META) of this table.  We are looking for
2788      * holes, overlaps, and cycles.
2789      * @return false if there are errors
2790      * @throws IOException
2791      */
2792     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2793       // When table is disabled no need to check for the region chain. Some of the regions
2794       // accidently if deployed, this below code might report some issues like missing start
2795       // or end regions or region hole in chain and may try to fix which is unwanted.
2796       if (isTableDisabled(this.tableName)) {
2797         return true;
2798       }
2799       int originalErrorsCount = errors.getErrorList().size();
2800       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2801       SortedSet<byte[]> splits = sc.getSplits();
2802 
2803       byte[] prevKey = null;
2804       byte[] problemKey = null;
2805 
2806       if (splits.size() == 0) {
2807         // no region for this table
2808         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2809       }
2810 
2811       for (byte[] key : splits) {
2812         Collection<HbckInfo> ranges = regions.get(key);
2813         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2814           for (HbckInfo rng : ranges) {
2815             handler.handleRegionStartKeyNotEmpty(rng);
2816           }
2817         }
2818 
2819         // check for degenerate ranges
2820         for (HbckInfo rng : ranges) {
2821           // special endkey case converts '' to null
2822           byte[] endKey = rng.getEndKey();
2823           endKey = (endKey.length == 0) ? null : endKey;
2824           if (Bytes.equals(rng.getStartKey(),endKey)) {
2825             handler.handleDegenerateRegion(rng);
2826           }
2827         }
2828 
2829         if (ranges.size() == 1) {
2830           // this split key is ok -- no overlap, not a hole.
2831           if (problemKey != null) {
2832             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2833           }
2834           problemKey = null; // fell through, no more problem.
2835         } else if (ranges.size() > 1) {
2836           // set the new problem key group name, if already have problem key, just
2837           // keep using it.
2838           if (problemKey == null) {
2839             // only for overlap regions.
2840             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2841             problemKey = key;
2842           }
2843           overlapGroups.putAll(problemKey, ranges);
2844 
2845           // record errors
2846           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
2847           //  this dumb and n^2 but this shouldn't happen often
2848           for (HbckInfo r1 : ranges) {
2849             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2850             subRange.remove(r1);
2851             for (HbckInfo r2 : subRange) {
2852               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2853               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
2854                 handler.handleDuplicateStartKeys(r1,r2);
2855               } else {
2856                 // overlap
2857                 handler.handleOverlapInRegionChain(r1, r2);
2858               }
2859             }
2860           }
2861 
2862         } else if (ranges.size() == 0) {
2863           if (problemKey != null) {
2864             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2865           }
2866           problemKey = null;
2867 
2868           byte[] holeStopKey = sc.getSplits().higher(key);
2869           // if higher key is null we reached the top.
2870           if (holeStopKey != null) {
2871             // hole
2872             handler.handleHoleInRegionChain(key, holeStopKey);
2873           }
2874         }
2875         prevKey = key;
2876       }
2877 
2878       // When the last region of a table is proper and having an empty end key, 'prevKey'
2879       // will be null.
2880       if (prevKey != null) {
2881         handler.handleRegionEndKeyNotEmpty(prevKey);
2882       }
2883 
2884       // TODO fold this into the TableIntegrityHandler
2885       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
2886         LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
2887             " false to run serially.");
2888         boolean ok = handleOverlapsParallel(handler, prevKey);
2889         if (!ok) {
2890           return false;
2891         }
2892       } else {
2893         LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
2894             " true to run in parallel.");
2895         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2896           handler.handleOverlapGroup(overlap);
2897         }
2898       }
2899 
2900       if (details) {
2901         // do full region split map dump
2902         errors.print("---- Table '"  +  this.tableName
2903             + "': region split map");
2904         dump(splits, regions);
2905         errors.print("---- Table '"  +  this.tableName
2906             + "': overlap groups");
2907         dumpOverlapProblems(overlapGroups);
2908         errors.print("There are " + overlapGroups.keySet().size()
2909             + " overlap groups with " + overlapGroups.size()
2910             + " overlapping regions");
2911       }
2912       if (!sidelinedRegions.isEmpty()) {
2913         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
2914         errors.print("---- Table '"  +  this.tableName
2915             + "': sidelined big overlapped regions");
2916         dumpSidelinedRegions(sidelinedRegions);
2917       }
2918       return errors.getErrorList().size() == originalErrorsCount;
2919     }
2920 
2921     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
2922         throws IOException {
2923       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
2924       // safely assume each group is independent.
2925       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
2926       List<Future<Void>> rets;
2927       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2928         //
2929         merges.add(new WorkItemOverlapMerge(overlap, handler));
2930       }
2931       try {
2932         rets = executor.invokeAll(merges);
2933       } catch (InterruptedException e) {
2934         LOG.error("Overlap merges were interrupted", e);
2935         return false;
2936       }
2937       for(int i=0; i<merges.size(); i++) {
2938         WorkItemOverlapMerge work = merges.get(i);
2939         Future<Void> f = rets.get(i);
2940         try {
2941           f.get();
2942         } catch(ExecutionException e) {
2943           LOG.warn("Failed to merge overlap group" + work, e.getCause());
2944         } catch (InterruptedException e) {
2945           LOG.error("Waiting for overlap merges was interrupted", e);
2946           return false;
2947         }
2948       }
2949       return true;
2950     }
2951 
2952     /**
2953      * This dumps data in a visually reasonable way for visual debugging
2954      *
2955      * @param splits
2956      * @param regions
2957      */
2958     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
2959       // we display this way because the last end key should be displayed as well.
2960       StringBuilder sb = new StringBuilder();
2961       for (byte[] k : splits) {
2962         sb.setLength(0); // clear out existing buffer, if any.
2963         sb.append(Bytes.toStringBinary(k) + ":\t");
2964         for (HbckInfo r : regions.get(k)) {
2965           sb.append("[ "+ r.toString() + ", "
2966               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
2967         }
2968         errors.print(sb.toString());
2969       }
2970     }
2971   }
2972 
2973   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
2974     // we display this way because the last end key should be displayed as
2975     // well.
2976     for (byte[] k : regions.keySet()) {
2977       errors.print(Bytes.toStringBinary(k) + ":");
2978       for (HbckInfo r : regions.get(k)) {
2979         errors.print("[ " + r.toString() + ", "
2980             + Bytes.toStringBinary(r.getEndKey()) + "]");
2981       }
2982       errors.print("----");
2983     }
2984   }
2985 
2986   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
2987     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
2988       TableName tableName = entry.getValue().getTableName();
2989       Path path = entry.getKey();
2990       errors.print("This sidelined region dir should be bulk loaded: "
2991         + path.toString());
2992       errors.print("Bulk load command looks like: "
2993         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
2994         + path.toUri().getPath() + " "+ tableName);
2995     }
2996   }
2997 
2998   public Multimap<byte[], HbckInfo> getOverlapGroups(
2999       TableName table) {
3000     TableInfo ti = tablesInfo.get(table);
3001     return ti.overlapGroups;
3002   }
3003 
3004   /**
3005    * Return a list of user-space table names whose metadata have not been
3006    * modified in the last few milliseconds specified by timelag
3007    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
3008    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
3009    * milliseconds specified by timelag, then the table is a candidate to be returned.
3010    * @return tables that have not been modified recently
3011    * @throws IOException if an error is encountered
3012    */
3013   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3014     List<TableName> tableNames = new ArrayList<TableName>();
3015     long now = System.currentTimeMillis();
3016 
3017     for (HbckInfo hbi : regionInfoMap.values()) {
3018       MetaEntry info = hbi.metaEntry;
3019 
3020       // if the start key is zero, then we have found the first region of a table.
3021       // pick only those tables that were not modified in the last few milliseconds.
3022       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3023         if (info.modTime + timelag < now) {
3024           tableNames.add(info.getTable());
3025         } else {
3026           numSkipped.incrementAndGet(); // one more in-flux table
3027         }
3028       }
3029     }
3030     return getHTableDescriptors(tableNames);
3031   }
3032 
3033   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3034     HTableDescriptor[] htd = new HTableDescriptor[0];
3035     Admin admin = null;
3036     try {
3037       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3038       admin = new HBaseAdmin(getConf());
3039       htd = admin.getTableDescriptorsByTableName(tableNames);
3040     } catch (IOException e) {
3041       LOG.debug("Exception getting table descriptors", e);
3042     } finally {
3043       if (admin != null) {
3044         try {
3045           admin.close();
3046         } catch (IOException e) {
3047           LOG.debug("Exception closing HBaseAdmin", e);
3048         }
3049       }
3050     }
3051     return htd;
3052   }
3053 
3054   /**
3055    * Gets the entry in regionInfo corresponding to the the given encoded
3056    * region name. If the region has not been seen yet, a new entry is added
3057    * and returned.
3058    */
3059   private synchronized HbckInfo getOrCreateInfo(String name) {
3060     HbckInfo hbi = regionInfoMap.get(name);
3061     if (hbi == null) {
3062       hbi = new HbckInfo(null);
3063       regionInfoMap.put(name, hbi);
3064     }
3065     return hbi;
3066   }
3067 
3068   private void checkAndFixTableLocks() throws IOException {
3069     TableLockChecker checker = new TableLockChecker(createZooKeeperWatcher(), errors);
3070     checker.checkTableLocks();
3071 
3072     if (this.fixTableLocks) {
3073       checker.fixExpiredTableLocks();
3074     }
3075   }
3076 
3077   /**
3078     * Check values in regionInfo for hbase:meta
3079     * Check if zero or more than one regions with hbase:meta are found.
3080     * If there are inconsistencies (i.e. zero or more than one regions
3081     * pretend to be holding the hbase:meta) try to fix that and report an error.
3082     * @throws IOException from HBaseFsckRepair functions
3083     * @throws KeeperException
3084     * @throws InterruptedException
3085     */
3086   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3087     Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3088     for (HbckInfo value : regionInfoMap.values()) {
3089       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3090         metaRegions.put(value.getReplicaId(), value);
3091       }
3092     }
3093     int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3094         .getRegionReplication();
3095     boolean noProblem = true;
3096     // There will be always entries in regionInfoMap corresponding to hbase:meta & its replicas
3097     // Check the deployed servers. It should be exactly one server for each replica.
3098     for (int i = 0; i < metaReplication; i++) {
3099       HbckInfo metaHbckInfo = metaRegions.remove(i);
3100       List<ServerName> servers = new ArrayList<ServerName>();
3101       if (metaHbckInfo != null) {
3102         servers = metaHbckInfo.deployedOn;
3103       }
3104       if (servers.size() != 1) {
3105         noProblem = false;
3106         if (servers.size() == 0) {
3107           assignMetaReplica(i);
3108         } else if (servers.size() > 1) {
3109           errors
3110           .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3111                        metaHbckInfo.getReplicaId() + " is found on more than one region.");
3112           if (shouldFixAssignments()) {
3113             errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3114                          metaHbckInfo.getReplicaId() +"..");
3115             setShouldRerun();
3116             // try fix it (treat is a dupe assignment)
3117             HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3118           }
3119         }
3120       }
3121     }
3122     // unassign whatever is remaining in metaRegions. They are excess replicas.
3123     for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3124       noProblem = false;
3125       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3126           "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3127           ", deployed " + metaRegions.size());
3128       if (shouldFixAssignments()) {
3129         errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3130             " of hbase:meta..");
3131         setShouldRerun();
3132         unassignMetaReplica(entry.getValue());
3133       }
3134     }
3135     // if noProblem is false, rerun hbck with hopefully fixed META
3136     // if noProblem is true, no errors, so continue normally
3137     return noProblem;
3138   }
3139 
3140   private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3141   KeeperException {
3142     undeployRegions(hi);
3143     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3144     ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3145   }
3146 
3147   private void assignMetaReplica(int replicaId)
3148       throws IOException, KeeperException, InterruptedException {
3149     errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3150         replicaId +" is not found on any region.");
3151     if (shouldFixAssignments()) {
3152       errors.print("Trying to fix a problem with hbase:meta..");
3153       setShouldRerun();
3154       // try to fix it (treat it as unassigned region)
3155       HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3156           HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3157       HBaseFsckRepair.fixUnassigned(admin, h);
3158       HBaseFsckRepair.waitUntilAssigned(admin, h);
3159     }
3160   }
3161 
3162   /**
3163    * Scan hbase:meta, adding all regions found to the regionInfo map.
3164    * @throws IOException if an error is encountered
3165    */
3166   boolean loadMetaEntries() throws IOException {
3167     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
3168       int countRecord = 1;
3169 
3170       // comparator to sort KeyValues with latest modtime
3171       final Comparator<Cell> comp = new Comparator<Cell>() {
3172         @Override
3173         public int compare(Cell k1, Cell k2) {
3174           return (int)(k1.getTimestamp() - k2.getTimestamp());
3175         }
3176       };
3177 
3178       @Override
3179       public boolean processRow(Result result) throws IOException {
3180         try {
3181 
3182           // record the latest modification of this META record
3183           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
3184           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3185           if (rl == null) {
3186             emptyRegionInfoQualifiers.add(result);
3187             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3188               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3189             return true;
3190           }
3191           ServerName sn = null;
3192           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3193               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3194             emptyRegionInfoQualifiers.add(result);
3195             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3196               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3197             return true;
3198           }
3199           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3200           if (!(isTableIncluded(hri.getTable())
3201               || hri.isMetaRegion())) {
3202             return true;
3203           }
3204           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3205           for (HRegionLocation h : rl.getRegionLocations()) {
3206             if (h == null || h.getRegionInfo() == null) {
3207               continue;
3208             }
3209             sn = h.getServerName();
3210             hri = h.getRegionInfo();
3211 
3212             MetaEntry m = null;
3213             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3214               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3215             } else {
3216               m = new MetaEntry(hri, sn, ts, null, null);
3217             }
3218             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3219             if (previous == null) {
3220               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3221             } else if (previous.metaEntry == null) {
3222               previous.metaEntry = m;
3223             } else {
3224               throw new IOException("Two entries in hbase:meta are same " + previous);
3225             }
3226           }
3227           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3228           for (HRegionInfo mergeRegion : new HRegionInfo[] {
3229               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3230             if (mergeRegion != null) {
3231               // This region is already been merged
3232               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3233               hbInfo.setMerged(true);
3234             }
3235           }
3236 
3237           // show proof of progress to the user, once for every 100 records.
3238           if (countRecord % 100 == 0) {
3239             errors.progress();
3240           }
3241           countRecord++;
3242           return true;
3243         } catch (RuntimeException e) {
3244           LOG.error("Result=" + result);
3245           throw e;
3246         }
3247       }
3248     };
3249     if (!checkMetaOnly) {
3250       // Scan hbase:meta to pick up user regions
3251       MetaScanner.metaScan(connection, visitor);
3252     }
3253 
3254     errors.print("");
3255     return true;
3256   }
3257 
3258   /**
3259    * Stores the regioninfo entries scanned from META
3260    */
3261   static class MetaEntry extends HRegionInfo {
3262     ServerName regionServer;   // server hosting this region
3263     long modTime;          // timestamp of most recent modification metadata
3264     HRegionInfo splitA, splitB; //split daughters
3265 
3266     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3267       this(rinfo, regionServer, modTime, null, null);
3268     }
3269 
3270     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3271         HRegionInfo splitA, HRegionInfo splitB) {
3272       super(rinfo);
3273       this.regionServer = regionServer;
3274       this.modTime = modTime;
3275       this.splitA = splitA;
3276       this.splitB = splitB;
3277     }
3278 
3279     @Override
3280     public boolean equals(Object o) {
3281       boolean superEq = super.equals(o);
3282       if (!superEq) {
3283         return superEq;
3284       }
3285 
3286       MetaEntry me = (MetaEntry) o;
3287       if (!regionServer.equals(me.regionServer)) {
3288         return false;
3289       }
3290       return (modTime == me.modTime);
3291     }
3292 
3293     @Override
3294     public int hashCode() {
3295       int hash = Arrays.hashCode(getRegionName());
3296       hash ^= getRegionId();
3297       hash ^= Arrays.hashCode(getStartKey());
3298       hash ^= Arrays.hashCode(getEndKey());
3299       hash ^= Boolean.valueOf(isOffline()).hashCode();
3300       hash ^= getTable().hashCode();
3301       if (regionServer != null) {
3302         hash ^= regionServer.hashCode();
3303       }
3304       hash ^= modTime;
3305       return hash;
3306     }
3307   }
3308 
3309   /**
3310    * Stores the regioninfo entries from HDFS
3311    */
3312   static class HdfsEntry {
3313     HRegionInfo hri;
3314     Path hdfsRegionDir = null;
3315     long hdfsRegionDirModTime  = 0;
3316     boolean hdfsRegioninfoFilePresent = false;
3317     boolean hdfsOnlyEdits = false;
3318   }
3319 
3320   /**
3321    * Stores the regioninfo retrieved from Online region servers.
3322    */
3323   static class OnlineEntry {
3324     HRegionInfo hri;
3325     ServerName hsa;
3326 
3327     @Override
3328     public String toString() {
3329       return hsa.toString() + ";" + hri.getRegionNameAsString();
3330     }
3331   }
3332 
3333   /**
3334    * Maintain information about a particular region.  It gathers information
3335    * from three places -- HDFS, META, and region servers.
3336    */
3337   public static class HbckInfo implements KeyRange {
3338     private MetaEntry metaEntry = null; // info in META
3339     private HdfsEntry hdfsEntry = null; // info in HDFS
3340     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3341     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3342     private boolean skipChecks = false; // whether to skip further checks to this region info.
3343     private boolean isMerged = false;// whether this region has already been merged into another one
3344     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3345     private HRegionInfo primaryHRIForDeployedReplica = null;
3346 
3347     HbckInfo(MetaEntry metaEntry) {
3348       this.metaEntry = metaEntry;
3349     }
3350 
3351     public int getReplicaId() {
3352       if (metaEntry != null) return metaEntry.getReplicaId();
3353       return deployedReplicaId;
3354     }
3355 
3356     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3357       OnlineEntry rse = new OnlineEntry() ;
3358       rse.hri = hri;
3359       rse.hsa = server;
3360       this.deployedEntries.add(rse);
3361       this.deployedOn.add(server);
3362       // save the replicaId that we see deployed in the cluster
3363       this.deployedReplicaId = hri.getReplicaId();
3364       this.primaryHRIForDeployedReplica =
3365           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3366     }
3367 
3368     @Override
3369     public synchronized String toString() {
3370       StringBuilder sb = new StringBuilder();
3371       sb.append("{ meta => ");
3372       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3373       sb.append( ", hdfs => " + getHdfsRegionDir());
3374       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3375       sb.append( ", replicaId => " + getReplicaId());
3376       sb.append(" }");
3377       return sb.toString();
3378     }
3379 
3380     @Override
3381     public byte[] getStartKey() {
3382       if (this.metaEntry != null) {
3383         return this.metaEntry.getStartKey();
3384       } else if (this.hdfsEntry != null) {
3385         return this.hdfsEntry.hri.getStartKey();
3386       } else {
3387         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3388         return null;
3389       }
3390     }
3391 
3392     @Override
3393     public byte[] getEndKey() {
3394       if (this.metaEntry != null) {
3395         return this.metaEntry.getEndKey();
3396       } else if (this.hdfsEntry != null) {
3397         return this.hdfsEntry.hri.getEndKey();
3398       } else {
3399         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3400         return null;
3401       }
3402     }
3403 
3404     public TableName getTableName() {
3405       if (this.metaEntry != null) {
3406         return this.metaEntry.getTable();
3407       } else if (this.hdfsEntry != null) {
3408         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3409         // so we get the name from the Path
3410         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3411         return FSUtils.getTableName(tableDir);
3412       } else {
3413         // return the info from the first online/deployed hri
3414         for (OnlineEntry e : deployedEntries) {
3415           return e.hri.getTable();
3416         }
3417         return null;
3418       }
3419     }
3420 
3421     public String getRegionNameAsString() {
3422       if (metaEntry != null) {
3423         return metaEntry.getRegionNameAsString();
3424       } else if (hdfsEntry != null) {
3425         if (hdfsEntry.hri != null) {
3426           return hdfsEntry.hri.getRegionNameAsString();
3427         }
3428       } else {
3429         // return the info from the first online/deployed hri
3430         for (OnlineEntry e : deployedEntries) {
3431           return e.hri.getRegionNameAsString();
3432         }
3433       }
3434       return null;
3435     }
3436 
3437     public byte[] getRegionName() {
3438       if (metaEntry != null) {
3439         return metaEntry.getRegionName();
3440       } else if (hdfsEntry != null) {
3441         return hdfsEntry.hri.getRegionName();
3442       } else {
3443         // return the info from the first online/deployed hri
3444         for (OnlineEntry e : deployedEntries) {
3445           return e.hri.getRegionName();
3446         }
3447         return null;
3448       }
3449     }
3450 
3451     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3452       return primaryHRIForDeployedReplica;
3453     }
3454 
3455     Path getHdfsRegionDir() {
3456       if (hdfsEntry == null) {
3457         return null;
3458       }
3459       return hdfsEntry.hdfsRegionDir;
3460     }
3461 
3462     boolean containsOnlyHdfsEdits() {
3463       if (hdfsEntry == null) {
3464         return false;
3465       }
3466       return hdfsEntry.hdfsOnlyEdits;
3467     }
3468 
3469     boolean isHdfsRegioninfoPresent() {
3470       if (hdfsEntry == null) {
3471         return false;
3472       }
3473       return hdfsEntry.hdfsRegioninfoFilePresent;
3474     }
3475 
3476     long getModTime() {
3477       if (hdfsEntry == null) {
3478         return 0;
3479       }
3480       return hdfsEntry.hdfsRegionDirModTime;
3481     }
3482 
3483     HRegionInfo getHdfsHRI() {
3484       if (hdfsEntry == null) {
3485         return null;
3486       }
3487       return hdfsEntry.hri;
3488     }
3489 
3490     public void setSkipChecks(boolean skipChecks) {
3491       this.skipChecks = skipChecks;
3492     }
3493 
3494     public boolean isSkipChecks() {
3495       return skipChecks;
3496     }
3497 
3498     public void setMerged(boolean isMerged) {
3499       this.isMerged = isMerged;
3500     }
3501 
3502     public boolean isMerged() {
3503       return this.isMerged;
3504     }
3505   }
3506 
3507   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3508     @Override
3509     public int compare(HbckInfo l, HbckInfo r) {
3510       if (l == r) {
3511         // same instance
3512         return 0;
3513       }
3514 
3515       int tableCompare = l.getTableName().compareTo(r.getTableName());
3516       if (tableCompare != 0) {
3517         return tableCompare;
3518       }
3519 
3520       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3521           l.getStartKey(), r.getStartKey());
3522       if (startComparison != 0) {
3523         return startComparison;
3524       }
3525 
3526       // Special case for absolute endkey
3527       byte[] endKey = r.getEndKey();
3528       endKey = (endKey.length == 0) ? null : endKey;
3529       byte[] endKey2 = l.getEndKey();
3530       endKey2 = (endKey2.length == 0) ? null : endKey2;
3531       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3532           endKey2,  endKey);
3533 
3534       if (endComparison != 0) {
3535         return endComparison;
3536       }
3537 
3538       // use regionId as tiebreaker.
3539       // Null is considered after all possible values so make it bigger.
3540       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3541         return 0;
3542       }
3543       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3544         return 1;
3545       }
3546       // l.hdfsEntry must not be null
3547       if (r.hdfsEntry == null) {
3548         return -1;
3549       }
3550       // both l.hdfsEntry and r.hdfsEntry must not be null.
3551       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3552     }
3553   };
3554 
3555   /**
3556    * Prints summary of all tables found on the system.
3557    */
3558   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3559     StringBuilder sb = new StringBuilder();
3560     errors.print("Summary:");
3561     for (TableInfo tInfo : tablesInfo.values()) {
3562       if (errors.tableHasErrors(tInfo)) {
3563         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3564       } else {
3565         errors.print("  " + tInfo.getName() + " is okay.");
3566       }
3567       errors.print("    Number of regions: " + tInfo.getNumRegions());
3568       sb.setLength(0); // clear out existing buffer, if any.
3569       sb.append("    Deployed on: ");
3570       for (ServerName server : tInfo.deployedOn) {
3571         sb.append(" " + server.toString());
3572       }
3573       errors.print(sb.toString());
3574     }
3575   }
3576 
3577   static ErrorReporter getErrorReporter(
3578       final Configuration conf) throws ClassNotFoundException {
3579     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3580     return ReflectionUtils.newInstance(reporter, conf);
3581   }
3582 
3583   public interface ErrorReporter {
3584     enum ERROR_CODE {
3585       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3586       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META,
3587       NOT_DEPLOYED,
3588       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3589       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3590       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3591       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3592       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR, ORPHAN_TABLE_STATE,
3593       NO_TABLE_STATE
3594     }
3595     void clear();
3596     void report(String message);
3597     void reportError(String message);
3598     void reportError(ERROR_CODE errorCode, String message);
3599     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3600     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3601     void reportError(
3602       ERROR_CODE errorCode,
3603       String message,
3604       TableInfo table,
3605       HbckInfo info1,
3606       HbckInfo info2
3607     );
3608     int summarize();
3609     void detail(String details);
3610     ArrayList<ERROR_CODE> getErrorList();
3611     void progress();
3612     void print(String message);
3613     void resetErrors();
3614     boolean tableHasErrors(TableInfo table);
3615   }
3616 
3617   static class PrintingErrorReporter implements ErrorReporter {
3618     public int errorCount = 0;
3619     private int showProgress;
3620 
3621     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3622 
3623     // for use by unit tests to verify which errors were discovered
3624     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3625 
3626     @Override
3627     public void clear() {
3628       errorTables.clear();
3629       errorList.clear();
3630       errorCount = 0;
3631     }
3632 
3633     @Override
3634     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3635       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3636         System.err.println(message);
3637         return;
3638       }
3639 
3640       errorList.add(errorCode);
3641       if (!summary) {
3642         System.out.println("ERROR: " + message);
3643       }
3644       errorCount++;
3645       showProgress = 0;
3646     }
3647 
3648     @Override
3649     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3650       errorTables.add(table);
3651       reportError(errorCode, message);
3652     }
3653 
3654     @Override
3655     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3656                                          HbckInfo info) {
3657       errorTables.add(table);
3658       String reference = "(region " + info.getRegionNameAsString() + ")";
3659       reportError(errorCode, reference + " " + message);
3660     }
3661 
3662     @Override
3663     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3664                                          HbckInfo info1, HbckInfo info2) {
3665       errorTables.add(table);
3666       String reference = "(regions " + info1.getRegionNameAsString()
3667           + " and " + info2.getRegionNameAsString() + ")";
3668       reportError(errorCode, reference + " " + message);
3669     }
3670 
3671     @Override
3672     public synchronized void reportError(String message) {
3673       reportError(ERROR_CODE.UNKNOWN, message);
3674     }
3675 
3676     /**
3677      * Report error information, but do not increment the error count.  Intended for cases
3678      * where the actual error would have been reported previously.
3679      * @param message
3680      */
3681     @Override
3682     public synchronized void report(String message) {
3683       if (! summary) {
3684         System.out.println("ERROR: " + message);
3685       }
3686       showProgress = 0;
3687     }
3688 
3689     @Override
3690     public synchronized int summarize() {
3691       System.out.println(Integer.toString(errorCount) +
3692                          " inconsistencies detected.");
3693       if (errorCount == 0) {
3694         System.out.println("Status: OK");
3695         return 0;
3696       } else {
3697         System.out.println("Status: INCONSISTENT");
3698         return -1;
3699       }
3700     }
3701 
3702     @Override
3703     public ArrayList<ERROR_CODE> getErrorList() {
3704       return errorList;
3705     }
3706 
3707     @Override
3708     public synchronized void print(String message) {
3709       if (!summary) {
3710         System.out.println(message);
3711       }
3712     }
3713 
3714     @Override
3715     public boolean tableHasErrors(TableInfo table) {
3716       return errorTables.contains(table);
3717     }
3718 
3719     @Override
3720     public void resetErrors() {
3721       errorCount = 0;
3722     }
3723 
3724     @Override
3725     public synchronized void detail(String message) {
3726       if (details) {
3727         System.out.println(message);
3728       }
3729       showProgress = 0;
3730     }
3731 
3732     @Override
3733     public synchronized void progress() {
3734       if (showProgress++ == 10) {
3735         if (!summary) {
3736           System.out.print(".");
3737         }
3738         showProgress = 0;
3739       }
3740     }
3741   }
3742 
3743   /**
3744    * Contact a region server and get all information from it
3745    */
3746   static class WorkItemRegion implements Callable<Void> {
3747     private HBaseFsck hbck;
3748     private ServerName rsinfo;
3749     private ErrorReporter errors;
3750     private HConnection connection;
3751 
3752     WorkItemRegion(HBaseFsck hbck, ServerName info,
3753                    ErrorReporter errors, HConnection connection) {
3754       this.hbck = hbck;
3755       this.rsinfo = info;
3756       this.errors = errors;
3757       this.connection = connection;
3758     }
3759 
3760     @Override
3761     public synchronized Void call() throws IOException {
3762       errors.progress();
3763       try {
3764         BlockingInterface server = connection.getAdmin(rsinfo);
3765 
3766         // list all online regions from this region server
3767         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3768         regions = filterRegions(regions);
3769 
3770         if (details) {
3771           errors.detail("RegionServer: " + rsinfo.getServerName() +
3772                            " number of regions: " + regions.size());
3773           for (HRegionInfo rinfo: regions) {
3774             errors.detail("  " + rinfo.getRegionNameAsString() +
3775                              " id: " + rinfo.getRegionId() +
3776                              " encoded_name: " + rinfo.getEncodedName() +
3777                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3778                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3779           }
3780         }
3781 
3782         // check to see if the existence of this region matches the region in META
3783         for (HRegionInfo r:regions) {
3784           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3785           hbi.addServer(r, rsinfo);
3786         }
3787       } catch (IOException e) {          // unable to connect to the region server.
3788         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3789           " Unable to fetch region information. " + e);
3790         throw e;
3791       }
3792       return null;
3793     }
3794 
3795     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
3796       List<HRegionInfo> ret = Lists.newArrayList();
3797       for (HRegionInfo hri : regions) {
3798         if (hri.isMetaTable() || (!hbck.checkMetaOnly
3799             && hbck.isTableIncluded(hri.getTable()))) {
3800           ret.add(hri);
3801         }
3802       }
3803       return ret;
3804     }
3805   }
3806 
3807   /**
3808    * Contact hdfs and get all information about specified table directory into
3809    * regioninfo list.
3810    */
3811   static class WorkItemHdfsDir implements Callable<Void> {
3812     private HBaseFsck hbck;
3813     private FileStatus tableDir;
3814     private ErrorReporter errors;
3815     private FileSystem fs;
3816 
3817     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
3818                     FileStatus status) {
3819       this.hbck = hbck;
3820       this.fs = fs;
3821       this.tableDir = status;
3822       this.errors = errors;
3823     }
3824 
3825     @Override
3826     public synchronized Void call() throws IOException {
3827       try {
3828         // level 2: <HBASE_DIR>/<table>/*
3829         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3830         for (FileStatus regionDir : regionDirs) {
3831           String encodedName = regionDir.getPath().getName();
3832           // ignore directories that aren't hexadecimal
3833           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
3834             continue;
3835           }
3836 
3837           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3838           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
3839           HdfsEntry he = new HdfsEntry();
3840           synchronized (hbi) {
3841             if (hbi.getHdfsRegionDir() != null) {
3842               errors.print("Directory " + encodedName + " duplicate??" +
3843                            hbi.getHdfsRegionDir());
3844             }
3845 
3846             he.hdfsRegionDir = regionDir.getPath();
3847             he.hdfsRegionDirModTime = regionDir.getModificationTime();
3848             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
3849             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
3850             // we add to orphan list when we attempt to read .regioninfo
3851 
3852             // Set a flag if this region contains only edits
3853             // This is special case if a region is left after split
3854             he.hdfsOnlyEdits = true;
3855             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3856             Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
3857             for (FileStatus subDir : subDirs) {
3858               String sdName = subDir.getPath().getName();
3859               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3860                 he.hdfsOnlyEdits = false;
3861                 break;
3862               }
3863             }
3864             hbi.hdfsEntry = he;
3865           }
3866         }
3867       } catch (IOException e) {
3868         // unable to connect to the region server.
3869         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3870             + tableDir.getPath().getName()
3871             + " Unable to fetch region information. " + e);
3872         throw e;
3873       }
3874       return null;
3875     }
3876   }
3877 
3878   /**
3879    * Contact hdfs and get all information about specified table directory into
3880    * regioninfo list.
3881    */
3882   static class WorkItemHdfsRegionInfo implements Callable<Void> {
3883     private HbckInfo hbi;
3884     private HBaseFsck hbck;
3885     private ErrorReporter errors;
3886 
3887     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
3888       this.hbi = hbi;
3889       this.hbck = hbck;
3890       this.errors = errors;
3891     }
3892 
3893     @Override
3894     public synchronized Void call() throws IOException {
3895       // only load entries that haven't been loaded yet.
3896       if (hbi.getHdfsHRI() == null) {
3897         try {
3898           hbck.loadHdfsRegioninfo(hbi);
3899         } catch (IOException ioe) {
3900           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3901               + hbi.getTableName() + " in hdfs dir "
3902               + hbi.getHdfsRegionDir()
3903               + "!  It may be an invalid format or version file.  Treating as "
3904               + "an orphaned regiondir.";
3905           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3906           try {
3907             hbck.debugLsr(hbi.getHdfsRegionDir());
3908           } catch (IOException ioe2) {
3909             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3910             throw ioe2;
3911           }
3912           hbck.orphanHdfsDirs.add(hbi);
3913           throw ioe;
3914         }
3915       }
3916       return null;
3917     }
3918   };
3919 
3920   /**
3921    * Display the full report from fsck. This displays all live and dead region
3922    * servers, and all known regions.
3923    */
3924   public void setDisplayFullReport() {
3925     details = true;
3926   }
3927 
3928   /**
3929    * Set summary mode.
3930    * Print only summary of the tables and status (OK or INCONSISTENT)
3931    */
3932   void setSummary() {
3933     summary = true;
3934   }
3935 
3936   /**
3937    * Set hbase:meta check mode.
3938    * Print only info about hbase:meta table deployment/state
3939    */
3940   void setCheckMetaOnly() {
3941     checkMetaOnly = true;
3942   }
3943 
3944   /**
3945    * Set region boundaries check mode.
3946    */
3947   void setRegionBoundariesCheck() {
3948     checkRegionBoundaries = true;
3949   }
3950 
3951   /**
3952    * Set table locks fix mode.
3953    * Delete table locks held for a long time
3954    */
3955   public void setFixTableLocks(boolean shouldFix) {
3956     fixTableLocks = shouldFix;
3957     fixAny |= shouldFix;
3958   }
3959 
3960   /**
3961    * Check if we should rerun fsck again. This checks if we've tried to
3962    * fix something and we should rerun fsck tool again.
3963    * Display the full report from fsck. This displays all live and dead
3964    * region servers, and all known regions.
3965    */
3966   void setShouldRerun() {
3967     rerun = true;
3968   }
3969 
3970   boolean shouldRerun() {
3971     return rerun;
3972   }
3973 
3974   /**
3975    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3976    * found by fsck utility.
3977    */
3978   public void setFixAssignments(boolean shouldFix) {
3979     fixAssignments = shouldFix;
3980     fixAny |= shouldFix;
3981   }
3982 
3983   boolean shouldFixAssignments() {
3984     return fixAssignments;
3985   }
3986 
3987   public void setFixMeta(boolean shouldFix) {
3988     fixMeta = shouldFix;
3989     fixAny |= shouldFix;
3990   }
3991 
3992   boolean shouldFixMeta() {
3993     return fixMeta;
3994   }
3995 
3996   public void setFixEmptyMetaCells(boolean shouldFix) {
3997     fixEmptyMetaCells = shouldFix;
3998     fixAny |= shouldFix;
3999   }
4000 
4001   boolean shouldFixEmptyMetaCells() {
4002     return fixEmptyMetaCells;
4003   }
4004 
4005   public void setCheckHdfs(boolean checking) {
4006     checkHdfs = checking;
4007   }
4008 
4009   boolean shouldCheckHdfs() {
4010     return checkHdfs;
4011   }
4012 
4013   public void setFixHdfsHoles(boolean shouldFix) {
4014     fixHdfsHoles = shouldFix;
4015     fixAny |= shouldFix;
4016   }
4017 
4018   boolean shouldFixHdfsHoles() {
4019     return fixHdfsHoles;
4020   }
4021 
4022   public void setFixTableOrphans(boolean shouldFix) {
4023     fixTableOrphans = shouldFix;
4024     fixAny |= shouldFix;
4025   }
4026 
4027   boolean shouldFixTableOrphans() {
4028     return fixTableOrphans;
4029   }
4030 
4031   public void setFixHdfsOverlaps(boolean shouldFix) {
4032     fixHdfsOverlaps = shouldFix;
4033     fixAny |= shouldFix;
4034   }
4035 
4036   boolean shouldFixHdfsOverlaps() {
4037     return fixHdfsOverlaps;
4038   }
4039 
4040   public void setFixHdfsOrphans(boolean shouldFix) {
4041     fixHdfsOrphans = shouldFix;
4042     fixAny |= shouldFix;
4043   }
4044 
4045   boolean shouldFixHdfsOrphans() {
4046     return fixHdfsOrphans;
4047   }
4048 
4049   public void setFixVersionFile(boolean shouldFix) {
4050     fixVersionFile = shouldFix;
4051     fixAny |= shouldFix;
4052   }
4053 
4054   public boolean shouldFixVersionFile() {
4055     return fixVersionFile;
4056   }
4057 
4058   public void setSidelineBigOverlaps(boolean sbo) {
4059     this.sidelineBigOverlaps = sbo;
4060   }
4061 
4062   public boolean shouldSidelineBigOverlaps() {
4063     return sidelineBigOverlaps;
4064   }
4065 
4066   public void setFixSplitParents(boolean shouldFix) {
4067     fixSplitParents = shouldFix;
4068     fixAny |= shouldFix;
4069   }
4070 
4071   boolean shouldFixSplitParents() {
4072     return fixSplitParents;
4073   }
4074 
4075   public void setFixReferenceFiles(boolean shouldFix) {
4076     fixReferenceFiles = shouldFix;
4077     fixAny |= shouldFix;
4078   }
4079 
4080   boolean shouldFixReferenceFiles() {
4081     return fixReferenceFiles;
4082   }
4083 
4084   public boolean shouldIgnorePreCheckPermission() {
4085     return !fixAny || ignorePreCheckPermission;
4086   }
4087 
4088   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4089     this.ignorePreCheckPermission = ignorePreCheckPermission;
4090   }
4091 
4092   /**
4093    * @param mm maximum number of regions to merge into a single region.
4094    */
4095   public void setMaxMerge(int mm) {
4096     this.maxMerge = mm;
4097   }
4098 
4099   public int getMaxMerge() {
4100     return maxMerge;
4101   }
4102 
4103   public void setMaxOverlapsToSideline(int mo) {
4104     this.maxOverlapsToSideline = mo;
4105   }
4106 
4107   public int getMaxOverlapsToSideline() {
4108     return maxOverlapsToSideline;
4109   }
4110 
4111   /**
4112    * Only check/fix tables specified by the list,
4113    * Empty list means all tables are included.
4114    */
4115   boolean isTableIncluded(TableName table) {
4116     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4117   }
4118 
4119   public void includeTable(TableName table) {
4120     tablesIncluded.add(table);
4121   }
4122 
4123   Set<TableName> getIncludedTables() {
4124     return new HashSet<TableName>(tablesIncluded);
4125   }
4126 
4127   /**
4128    * We are interested in only those tables that have not changed their state in
4129    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
4130    * @param seconds - the time in seconds
4131    */
4132   public void setTimeLag(long seconds) {
4133     timelag = seconds * 1000; // convert to milliseconds
4134   }
4135 
4136   /**
4137    *
4138    * @param sidelineDir - HDFS path to sideline data
4139    */
4140   public void setSidelineDir(String sidelineDir) {
4141     this.sidelineDir = new Path(sidelineDir);
4142   }
4143 
4144   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4145     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4146   }
4147 
4148   public HFileCorruptionChecker getHFilecorruptionChecker() {
4149     return hfcc;
4150   }
4151 
4152   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4153     this.hfcc = hfcc;
4154   }
4155 
4156   public void setRetCode(int code) {
4157     this.retcode = code;
4158   }
4159 
4160   public int getRetCode() {
4161     return retcode;
4162   }
4163 
4164   protected HBaseFsck printUsageAndExit() {
4165     StringWriter sw = new StringWriter(2048);
4166     PrintWriter out = new PrintWriter(sw);
4167     out.println("Usage: fsck [opts] {only tables}");
4168     out.println(" where [opts] are:");
4169     out.println("   -help Display help options (this)");
4170     out.println("   -details Display full report of all regions.");
4171     out.println("   -timelag <timeInSeconds>  Process only regions that " +
4172                        " have not experienced any metadata updates in the last " +
4173                        " <timeInSeconds> seconds.");
4174     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4175         " before checking if the fix worked if run with -fix");
4176     out.println("   -summary Print only summary of the tables and status.");
4177     out.println("   -metaonly Only check the state of the hbase:meta table.");
4178     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4179     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
4180 
4181     out.println("");
4182     out.println("  Metadata Repair options: (expert features, use with caution!)");
4183     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
4184     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
4185     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
4186     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
4187         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4188     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
4189     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
4190     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4191     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
4192     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
4193     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4194     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
4195     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4196     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
4197     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
4198     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
4199     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
4200         + " (empty REGIONINFO_QUALIFIER rows)");
4201 
4202     out.println("");
4203     out.println("  Datafile Repair options: (expert features, use with caution!)");
4204     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
4205     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
4206 
4207     out.println("");
4208     out.println("  Metadata Repair shortcuts");
4209     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4210         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
4211     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4212 
4213     out.println("");
4214     out.println("  Table lock options");
4215     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4216 
4217     out.flush();
4218     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4219 
4220     setRetCode(-2);
4221     return this;
4222   }
4223 
4224   /**
4225    * Main program
4226    *
4227    * @param args
4228    * @throws Exception
4229    */
4230   public static void main(String[] args) throws Exception {
4231     // create a fsck object
4232     Configuration conf = HBaseConfiguration.create();
4233     Path hbasedir = FSUtils.getRootDir(conf);
4234     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4235     FSUtils.setFsDefault(conf, new Path(defaultFs));
4236     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4237     System.exit(ret);
4238   }
4239 
4240   /**
4241    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
4242    */
4243   static class HBaseFsckTool extends Configured implements Tool {
4244     HBaseFsckTool(Configuration conf) { super(conf); }
4245     @Override
4246     public int run(String[] args) throws Exception {
4247       HBaseFsck hbck = new HBaseFsck(getConf());
4248       hbck.exec(hbck.executor, args);
4249       hbck.close();
4250       return hbck.getRetCode();
4251     }
4252   };
4253 
4254 
4255   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4256     ServiceException, InterruptedException {
4257     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4258 
4259     boolean checkCorruptHFiles = false;
4260     boolean sidelineCorruptHFiles = false;
4261 
4262     // Process command-line args.
4263     for (int i = 0; i < args.length; i++) {
4264       String cmd = args[i];
4265       if (cmd.equals("-help") || cmd.equals("-h")) {
4266         return printUsageAndExit();
4267       } else if (cmd.equals("-details")) {
4268         setDisplayFullReport();
4269       } else if (cmd.equals("-timelag")) {
4270         if (i == args.length - 1) {
4271           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4272           return printUsageAndExit();
4273         }
4274         try {
4275           long timelag = Long.parseLong(args[i+1]);
4276           setTimeLag(timelag);
4277         } catch (NumberFormatException e) {
4278           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4279           return printUsageAndExit();
4280         }
4281         i++;
4282       } else if (cmd.equals("-sleepBeforeRerun")) {
4283         if (i == args.length - 1) {
4284           errors.reportError(ERROR_CODE.WRONG_USAGE,
4285             "HBaseFsck: -sleepBeforeRerun needs a value.");
4286           return printUsageAndExit();
4287         }
4288         try {
4289           sleepBeforeRerun = Long.parseLong(args[i+1]);
4290         } catch (NumberFormatException e) {
4291           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4292           return printUsageAndExit();
4293         }
4294         i++;
4295       } else if (cmd.equals("-sidelineDir")) {
4296         if (i == args.length - 1) {
4297           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4298           return printUsageAndExit();
4299         }
4300         i++;
4301         setSidelineDir(args[i]);
4302       } else if (cmd.equals("-fix")) {
4303         errors.reportError(ERROR_CODE.WRONG_USAGE,
4304           "This option is deprecated, please use  -fixAssignments instead.");
4305         setFixAssignments(true);
4306       } else if (cmd.equals("-fixAssignments")) {
4307         setFixAssignments(true);
4308       } else if (cmd.equals("-fixMeta")) {
4309         setFixMeta(true);
4310       } else if (cmd.equals("-noHdfsChecking")) {
4311         setCheckHdfs(false);
4312       } else if (cmd.equals("-fixHdfsHoles")) {
4313         setFixHdfsHoles(true);
4314       } else if (cmd.equals("-fixHdfsOrphans")) {
4315         setFixHdfsOrphans(true);
4316       } else if (cmd.equals("-fixTableOrphans")) {
4317         setFixTableOrphans(true);
4318       } else if (cmd.equals("-fixHdfsOverlaps")) {
4319         setFixHdfsOverlaps(true);
4320       } else if (cmd.equals("-fixVersionFile")) {
4321         setFixVersionFile(true);
4322       } else if (cmd.equals("-sidelineBigOverlaps")) {
4323         setSidelineBigOverlaps(true);
4324       } else if (cmd.equals("-fixSplitParents")) {
4325         setFixSplitParents(true);
4326       } else if (cmd.equals("-ignorePreCheckPermission")) {
4327         setIgnorePreCheckPermission(true);
4328       } else if (cmd.equals("-checkCorruptHFiles")) {
4329         checkCorruptHFiles = true;
4330       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4331         sidelineCorruptHFiles = true;
4332       } else if (cmd.equals("-fixReferenceFiles")) {
4333         setFixReferenceFiles(true);
4334       } else if (cmd.equals("-fixEmptyMetaCells")) {
4335         setFixEmptyMetaCells(true);
4336       } else if (cmd.equals("-repair")) {
4337         // this attempts to merge overlapping hdfs regions, needs testing
4338         // under load
4339         setFixHdfsHoles(true);
4340         setFixHdfsOrphans(true);
4341         setFixMeta(true);
4342         setFixAssignments(true);
4343         setFixHdfsOverlaps(true);
4344         setFixVersionFile(true);
4345         setSidelineBigOverlaps(true);
4346         setFixSplitParents(false);
4347         setCheckHdfs(true);
4348         setFixReferenceFiles(true);
4349         setFixTableLocks(true);
4350       } else if (cmd.equals("-repairHoles")) {
4351         // this will make all missing hdfs regions available but may lose data
4352         setFixHdfsHoles(true);
4353         setFixHdfsOrphans(false);
4354         setFixMeta(true);
4355         setFixAssignments(true);
4356         setFixHdfsOverlaps(false);
4357         setSidelineBigOverlaps(false);
4358         setFixSplitParents(false);
4359         setCheckHdfs(true);
4360       } else if (cmd.equals("-maxOverlapsToSideline")) {
4361         if (i == args.length - 1) {
4362           errors.reportError(ERROR_CODE.WRONG_USAGE,
4363             "-maxOverlapsToSideline needs a numeric value argument.");
4364           return printUsageAndExit();
4365         }
4366         try {
4367           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4368           setMaxOverlapsToSideline(maxOverlapsToSideline);
4369         } catch (NumberFormatException e) {
4370           errors.reportError(ERROR_CODE.WRONG_USAGE,
4371             "-maxOverlapsToSideline needs a numeric value argument.");
4372           return printUsageAndExit();
4373         }
4374         i++;
4375       } else if (cmd.equals("-maxMerge")) {
4376         if (i == args.length - 1) {
4377           errors.reportError(ERROR_CODE.WRONG_USAGE,
4378             "-maxMerge needs a numeric value argument.");
4379           return printUsageAndExit();
4380         }
4381         try {
4382           int maxMerge = Integer.parseInt(args[i+1]);
4383           setMaxMerge(maxMerge);
4384         } catch (NumberFormatException e) {
4385           errors.reportError(ERROR_CODE.WRONG_USAGE,
4386             "-maxMerge needs a numeric value argument.");
4387           return printUsageAndExit();
4388         }
4389         i++;
4390       } else if (cmd.equals("-summary")) {
4391         setSummary();
4392       } else if (cmd.equals("-metaonly")) {
4393         setCheckMetaOnly();
4394       } else if (cmd.equals("-boundaries")) {
4395         setRegionBoundariesCheck();
4396       } else if (cmd.equals("-fixTableLocks")) {
4397         setFixTableLocks(true);
4398       } else if (cmd.startsWith("-")) {
4399         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4400         return printUsageAndExit();
4401       } else {
4402         includeTable(TableName.valueOf(cmd));
4403         errors.print("Allow checking/fixes for table: " + cmd);
4404       }
4405     }
4406 
4407     errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4408 
4409     // pre-check current user has FS write permission or not
4410     try {
4411       preCheckPermission();
4412     } catch (AccessDeniedException ace) {
4413       Runtime.getRuntime().exit(-1);
4414     } catch (IOException ioe) {
4415       Runtime.getRuntime().exit(-1);
4416     }
4417 
4418     // do the real work of hbck
4419     connect();
4420 
4421     try {
4422       // if corrupt file mode is on, first fix them since they may be opened later
4423       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4424         LOG.info("Checking all hfiles for corruption");
4425         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4426         setHFileCorruptionChecker(hfcc); // so we can get result
4427         Collection<TableName> tables = getIncludedTables();
4428         Collection<Path> tableDirs = new ArrayList<Path>();
4429         Path rootdir = FSUtils.getRootDir(getConf());
4430         if (tables.size() > 0) {
4431           for (TableName t : tables) {
4432             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4433           }
4434         } else {
4435           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4436         }
4437         hfcc.checkTables(tableDirs);
4438         hfcc.report(errors);
4439       }
4440 
4441       // check and fix table integrity, region consistency.
4442       int code = onlineHbck();
4443       setRetCode(code);
4444       // If we have changed the HBase state it is better to run hbck again
4445       // to see if we haven't broken something else in the process.
4446       // We run it only once more because otherwise we can easily fall into
4447       // an infinite loop.
4448       if (shouldRerun()) {
4449         try {
4450           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4451           Thread.sleep(sleepBeforeRerun);
4452         } catch (InterruptedException ie) {
4453           LOG.warn("Interrupted while sleeping");
4454           return this;
4455         }
4456         // Just report
4457         setFixAssignments(false);
4458         setFixMeta(false);
4459         setFixHdfsHoles(false);
4460         setFixHdfsOverlaps(false);
4461         setFixVersionFile(false);
4462         setFixTableOrphans(false);
4463         errors.resetErrors();
4464         code = onlineHbck();
4465         setRetCode(code);
4466       }
4467     } finally {
4468       IOUtils.cleanup(null, this);
4469     }
4470     return this;
4471   }
4472 
4473   /**
4474    * ls -r for debugging purposes
4475    */
4476   void debugLsr(Path p) throws IOException {
4477     debugLsr(getConf(), p, errors);
4478   }
4479 
4480   /**
4481    * ls -r for debugging purposes
4482    */
4483   public static void debugLsr(Configuration conf,
4484       Path p) throws IOException {
4485     debugLsr(conf, p, new PrintingErrorReporter());
4486   }
4487 
4488   /**
4489    * ls -r for debugging purposes
4490    */
4491   public static void debugLsr(Configuration conf,
4492       Path p, ErrorReporter errors) throws IOException {
4493     if (!LOG.isDebugEnabled() || p == null) {
4494       return;
4495     }
4496     FileSystem fs = p.getFileSystem(conf);
4497 
4498     if (!fs.exists(p)) {
4499       // nothing
4500       return;
4501     }
4502     errors.print(p.toString());
4503 
4504     if (fs.isFile(p)) {
4505       return;
4506     }
4507 
4508     if (fs.getFileStatus(p).isDirectory()) {
4509       FileStatus[] fss= fs.listStatus(p);
4510       for (FileStatus status : fss) {
4511         debugLsr(conf, status.getPath(), errors);
4512       }
4513     }
4514   }
4515 }