001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.util; 019 020import java.io.Closeable; 021import java.io.FileNotFoundException; 022import java.io.IOException; 023import java.io.InterruptedIOException; 024import java.io.PrintWriter; 025import java.io.StringWriter; 026import java.net.InetAddress; 027import java.net.URI; 028import java.util.ArrayList; 029import java.util.Collection; 030import java.util.Collections; 031import java.util.Comparator; 032import java.util.EnumSet; 033import java.util.HashMap; 034import java.util.HashSet; 035import java.util.Iterator; 036import java.util.List; 037import java.util.Locale; 038import java.util.Map; 039import java.util.Map.Entry; 040import java.util.Objects; 041import java.util.Optional; 042import java.util.Set; 043import java.util.SortedMap; 044import java.util.TreeMap; 045import java.util.Vector; 046import java.util.concurrent.Callable; 047import java.util.concurrent.ConcurrentSkipListMap; 048import java.util.concurrent.ExecutionException; 049import java.util.concurrent.ExecutorService; 050import java.util.concurrent.Executors; 051import java.util.concurrent.Future; 052import java.util.concurrent.FutureTask; 053import java.util.concurrent.ScheduledThreadPoolExecutor; 054import java.util.concurrent.TimeUnit; 055import java.util.concurrent.TimeoutException; 056import java.util.concurrent.atomic.AtomicBoolean; 057import java.util.concurrent.atomic.AtomicInteger; 058import java.util.stream.Collectors; 059 060import org.apache.commons.io.IOUtils; 061import org.apache.commons.lang3.StringUtils; 062import org.apache.hadoop.conf.Configuration; 063import org.apache.hadoop.conf.Configured; 064import org.apache.hadoop.fs.FSDataOutputStream; 065import org.apache.hadoop.fs.FileStatus; 066import org.apache.hadoop.fs.FileSystem; 067import org.apache.hadoop.fs.Path; 068import org.apache.hadoop.fs.permission.FsAction; 069import org.apache.hadoop.fs.permission.FsPermission; 070import org.apache.hadoop.hbase.Abortable; 071import org.apache.hadoop.hbase.Cell; 072import org.apache.hadoop.hbase.CellUtil; 073import org.apache.hadoop.hbase.ClusterMetrics; 074import org.apache.hadoop.hbase.ClusterMetrics.Option; 075import org.apache.hadoop.hbase.HBaseConfiguration; 076import org.apache.hadoop.hbase.HBaseInterfaceAudience; 077import org.apache.hadoop.hbase.HConstants; 078import org.apache.hadoop.hbase.HRegionLocation; 079import org.apache.hadoop.hbase.KeyValue; 080import org.apache.hadoop.hbase.MasterNotRunningException; 081import org.apache.hadoop.hbase.MetaTableAccessor; 082import org.apache.hadoop.hbase.RegionLocations; 083import org.apache.hadoop.hbase.ServerName; 084import org.apache.hadoop.hbase.TableName; 085import org.apache.hadoop.hbase.TableNotFoundException; 086import org.apache.hadoop.hbase.ZooKeeperConnectionException; 087import org.apache.hadoop.hbase.client.Admin; 088import org.apache.hadoop.hbase.client.ClusterConnection; 089import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 090import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 091import org.apache.hadoop.hbase.client.Connection; 092import org.apache.hadoop.hbase.client.ConnectionFactory; 093import org.apache.hadoop.hbase.client.Delete; 094import org.apache.hadoop.hbase.client.Get; 095import org.apache.hadoop.hbase.client.Put; 096import org.apache.hadoop.hbase.client.RegionInfo; 097import org.apache.hadoop.hbase.client.RegionInfoBuilder; 098import org.apache.hadoop.hbase.client.RegionReplicaUtil; 099import org.apache.hadoop.hbase.client.Result; 100import org.apache.hadoop.hbase.client.ResultScanner; 101import org.apache.hadoop.hbase.client.RowMutations; 102import org.apache.hadoop.hbase.client.Scan; 103import org.apache.hadoop.hbase.client.Table; 104import org.apache.hadoop.hbase.client.TableDescriptor; 105import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 106import org.apache.hadoop.hbase.client.TableState; 107import org.apache.hadoop.hbase.io.FileLink; 108import org.apache.hadoop.hbase.io.HFileLink; 109import org.apache.hadoop.hbase.io.hfile.CacheConfig; 110import org.apache.hadoop.hbase.io.hfile.HFile; 111import org.apache.hadoop.hbase.master.RegionState; 112import org.apache.hadoop.hbase.regionserver.HRegion; 113import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 114import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 115import org.apache.hadoop.hbase.replication.ReplicationException; 116import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; 117import org.apache.hadoop.hbase.replication.ReplicationQueueStorage; 118import org.apache.hadoop.hbase.replication.ReplicationStorageFactory; 119import org.apache.hadoop.hbase.security.AccessDeniedException; 120import org.apache.hadoop.hbase.security.UserProvider; 121import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator; 122import org.apache.hadoop.hbase.util.HbckErrorReporter.ERROR_CODE; 123import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; 124import org.apache.hadoop.hbase.util.hbck.ReplicationChecker; 125import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler; 126import org.apache.hadoop.hbase.wal.WALSplitter; 127import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 128import org.apache.hadoop.hbase.zookeeper.ZKUtil; 129import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 130import org.apache.hadoop.hbase.zookeeper.ZNodePaths; 131import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; 132import org.apache.hadoop.ipc.RemoteException; 133import org.apache.hadoop.security.UserGroupInformation; 134import org.apache.hadoop.util.ReflectionUtils; 135import org.apache.hadoop.util.Tool; 136import org.apache.hadoop.util.ToolRunner; 137import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; 138import org.apache.hbase.thirdparty.com.google.common.collect.Sets; 139import org.apache.yetus.audience.InterfaceAudience; 140import org.apache.yetus.audience.InterfaceStability; 141import org.apache.zookeeper.KeeperException; 142import org.slf4j.Logger; 143import org.slf4j.LoggerFactory; 144 145import org.apache.hbase.thirdparty.com.google.common.base.Joiner; 146import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; 147import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 148 149import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 150import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface; 151 152/** 153 * HBaseFsck (hbck) is a tool for checking and repairing region consistency and 154 * table integrity problems in a corrupted HBase. This tool was written for hbase-1.x. It does not 155 * work with hbase-2.x; it can read state but is not allowed to change state; i.e. effect 'repair'. 156 * Even though it can 'read' state, given how so much has changed in how hbase1 and hbase2 operate, 157 * it will often misread. See hbck2 (HBASE-19121) for a hbck tool for hbase2. This class is 158 * deprecated. 159 * 160 * <p> 161 * Region consistency checks verify that hbase:meta, region deployment on region 162 * servers and the state of data in HDFS (.regioninfo files) all are in 163 * accordance. 164 * <p> 165 * Table integrity checks verify that all possible row keys resolve to exactly 166 * one region of a table. This means there are no individual degenerate 167 * or backwards regions; no holes between regions; and that there are no 168 * overlapping regions. 169 * <p> 170 * The general repair strategy works in two phases: 171 * <ol> 172 * <li> Repair Table Integrity on HDFS. (merge or fabricate regions) 173 * <li> Repair Region Consistency with hbase:meta and assignments 174 * </ol> 175 * <p> 176 * For table integrity repairs, the tables' region directories are scanned 177 * for .regioninfo files. Each table's integrity is then verified. If there 178 * are any orphan regions (regions with no .regioninfo files) or holes, new 179 * regions are fabricated. Backwards regions are sidelined as well as empty 180 * degenerate (endkey==startkey) regions. If there are any overlapping regions, 181 * a new region is created and all data is merged into the new region. 182 * <p> 183 * Table integrity repairs deal solely with HDFS and could potentially be done 184 * offline -- the hbase region servers or master do not need to be running. 185 * This phase can eventually be used to completely reconstruct the hbase:meta table in 186 * an offline fashion. 187 * <p> 188 * Region consistency requires three conditions -- 1) valid .regioninfo file 189 * present in an HDFS region dir, 2) valid row with .regioninfo data in META, 190 * and 3) a region is deployed only at the regionserver that was assigned to 191 * with proper state in the master. 192 * <p> 193 * Region consistency repairs require hbase to be online so that hbck can 194 * contact the HBase master and region servers. The hbck#connect() method must 195 * first be called successfully. Much of the region consistency information 196 * is transient and less risky to repair. 197 * <p> 198 * If hbck is run from the command line, there are a handful of arguments that 199 * can be used to limit the kinds of repairs hbck will do. See the code in 200 * {@link #printUsageAndExit()} for more details. 201 * @deprecated For removal in hbase-4.0.0. Use HBCK2 instead. 202 */ 203@Deprecated 204@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) 205@InterfaceStability.Evolving 206public class HBaseFsck extends Configured implements Closeable { 207 public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute 208 public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000; 209 private static final int MAX_NUM_THREADS = 50; // #threads to contact regions 210 private static boolean rsSupportsOffline = true; 211 private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2; 212 private static final int DEFAULT_MAX_MERGE = 5; 213 214 /** 215 * Here is where hbase-1.x used to default the lock for hbck1. 216 * It puts in place a lock when it goes to write/make changes. 217 */ 218 @VisibleForTesting 219 public static final String HBCK_LOCK_FILE = "hbase-hbck.lock"; 220 private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5; 221 private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds 222 private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds 223 // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD. 224 // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for 225 // AlreadyBeingCreatedException which is implies timeout on this operations up to 226 // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds). 227 private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds 228 private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5; 229 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds 230 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds 231 232 /********************** 233 * Internal resources 234 **********************/ 235 private static final Logger LOG = LoggerFactory.getLogger(HBaseFsck.class.getName()); 236 private ClusterMetrics status; 237 private ClusterConnection connection; 238 private Admin admin; 239 private Table meta; 240 // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions 241 protected ExecutorService executor; 242 private long startMillis = EnvironmentEdgeManager.currentTime(); 243 private HFileCorruptionChecker hfcc; 244 private int retcode = 0; 245 private Path HBCK_LOCK_PATH; 246 private FSDataOutputStream hbckOutFd; 247 // This lock is to prevent cleanup of balancer resources twice between 248 // ShutdownHook and the main code. We cleanup only if the connect() is 249 // successful 250 private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false); 251 252 // Unsupported options in HBase 2.0+ 253 private static final Set<String> unsupportedOptionsInV2 = Sets.newHashSet("-fix", 254 "-fixAssignments", "-fixMeta", "-fixHdfsHoles", "-fixHdfsOrphans", "-fixTableOrphans", 255 "-fixHdfsOverlaps", "-sidelineBigOverlaps", "-fixSplitParents", "-removeParents", 256 "-fixEmptyMetaCells", "-repair", "-repairHoles", "-maxOverlapsToSideline", "-maxMerge"); 257 258 /*********** 259 * Options 260 ***********/ 261 private static boolean details = false; // do we display the full report 262 private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older 263 private static boolean forceExclusive = false; // only this hbck can modify HBase 264 private boolean fixAssignments = false; // fix assignment errors? 265 private boolean fixMeta = false; // fix meta errors? 266 private boolean checkHdfs = true; // load and check fs consistency? 267 private boolean fixHdfsHoles = false; // fix fs holes? 268 private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky) 269 private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo) 270 private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo) 271 private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs 272 private boolean fixSplitParents = false; // fix lingering split parents 273 private boolean removeParents = false; // remove split parents 274 private boolean fixReferenceFiles = false; // fix lingering reference store file 275 private boolean fixHFileLinks = false; // fix lingering HFileLinks 276 private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows 277 private boolean fixReplication = false; // fix undeleted replication queues for removed peer 278 private boolean cleanReplicationBarrier = false; // clean replication barriers of a table 279 private boolean fixAny = false; // Set to true if any of the fix is required. 280 281 // limit checking/fixes to listed tables, if empty attempt to check/fix all 282 // hbase:meta are always checked 283 private Set<TableName> tablesIncluded = new HashSet<>(); 284 private TableName cleanReplicationBarrierTable; 285 private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge 286 // maximum number of overlapping regions to sideline 287 private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; 288 private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions 289 private Path sidelineDir = null; 290 291 private boolean rerun = false; // if we tried to fix something, rerun hbck 292 private static boolean summary = false; // if we want to print less output 293 private boolean checkMetaOnly = false; 294 private boolean checkRegionBoundaries = false; 295 private boolean ignorePreCheckPermission = false; // if pre-check permission 296 297 /********* 298 * State 299 *********/ 300 final private HbckErrorReporter errors; 301 int fixes = 0; 302 303 /** 304 * This map contains the state of all hbck items. It maps from encoded region 305 * name to HbckRegionInfo structure. The information contained in HbckRegionInfo is used 306 * to detect and correct consistency (hdfs/meta/deployment) problems. 307 */ 308 private TreeMap<String, HbckRegionInfo> regionInfoMap = new TreeMap<>(); 309 // Empty regioninfo qualifiers in hbase:meta 310 private Set<Result> emptyRegionInfoQualifiers = new HashSet<>(); 311 312 /** 313 * This map from Tablename -> TableInfo contains the structures necessary to 314 * detect table consistency problems (holes, dupes, overlaps). It is sorted 315 * to prevent dupes. 316 * 317 * If tablesIncluded is empty, this map contains all tables. 318 * Otherwise, it contains only meta tables and tables in tablesIncluded, 319 * unless checkMetaOnly is specified, in which case, it contains only 320 * the meta table 321 */ 322 private SortedMap<TableName, HbckTableInfo> tablesInfo = new ConcurrentSkipListMap<>(); 323 324 /** 325 * When initially looking at HDFS, we attempt to find any orphaned data. 326 */ 327 private List<HbckRegionInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<>()); 328 329 private Map<TableName, Set<String>> orphanTableDirs = new HashMap<>(); 330 private Map<TableName, TableState> tableStates = new HashMap<>(); 331 private final RetryCounterFactory lockFileRetryCounterFactory; 332 private final RetryCounterFactory createZNodeRetryCounterFactory; 333 334 private Map<TableName, Set<String>> skippedRegions = new HashMap<>(); 335 336 private ZKWatcher zkw = null; 337 private String hbckEphemeralNodePath = null; 338 private boolean hbckZodeCreated = false; 339 340 /** 341 * Constructor 342 * 343 * @param conf Configuration object 344 * @throws MasterNotRunningException if the master is not running 345 * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper 346 */ 347 public HBaseFsck(Configuration conf) throws IOException, ClassNotFoundException { 348 this(conf, createThreadPool(conf)); 349 } 350 351 private static ExecutorService createThreadPool(Configuration conf) { 352 int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS); 353 return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck")); 354 } 355 356 /** 357 * Constructor 358 * 359 * @param conf 360 * Configuration object 361 * @throws MasterNotRunningException 362 * if the master is not running 363 * @throws ZooKeeperConnectionException 364 * if unable to connect to ZooKeeper 365 */ 366 public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException, 367 ZooKeeperConnectionException, IOException, ClassNotFoundException { 368 super(conf); 369 errors = getErrorReporter(getConf()); 370 this.executor = exec; 371 lockFileRetryCounterFactory = createLockRetryCounterFactory(getConf()); 372 createZNodeRetryCounterFactory = createZnodeRetryCounterFactory(getConf()); 373 zkw = createZooKeeperWatcher(); 374 } 375 376 /** 377 * @return A retry counter factory configured for retrying lock file creation. 378 */ 379 public static RetryCounterFactory createLockRetryCounterFactory(Configuration conf) { 380 return new RetryCounterFactory( 381 conf.getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS), 382 conf.getInt("hbase.hbck.lockfile.attempt.sleep.interval", 383 DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL), 384 conf.getInt("hbase.hbck.lockfile.attempt.maxsleeptime", 385 DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME)); 386 } 387 388 /** 389 * @return A retry counter factory configured for retrying znode creation. 390 */ 391 private static RetryCounterFactory createZnodeRetryCounterFactory(Configuration conf) { 392 return new RetryCounterFactory( 393 conf.getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS), 394 conf.getInt("hbase.hbck.createznode.attempt.sleep.interval", 395 DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL), 396 conf.getInt("hbase.hbck.createznode.attempt.maxsleeptime", 397 DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME)); 398 } 399 400 /** 401 * @return Return the tmp dir this tool writes too. 402 */ 403 @VisibleForTesting 404 public static Path getTmpDir(Configuration conf) throws IOException { 405 return new Path(FSUtils.getRootDir(conf), HConstants.HBASE_TEMP_DIRECTORY); 406 } 407 408 private static class FileLockCallable implements Callable<FSDataOutputStream> { 409 RetryCounter retryCounter; 410 private final Configuration conf; 411 private Path hbckLockPath = null; 412 413 public FileLockCallable(Configuration conf, RetryCounter retryCounter) { 414 this.retryCounter = retryCounter; 415 this.conf = conf; 416 } 417 418 /** 419 * @return Will be <code>null</code> unless you call {@link #call()} 420 */ 421 Path getHbckLockPath() { 422 return this.hbckLockPath; 423 } 424 425 @Override 426 public FSDataOutputStream call() throws IOException { 427 try { 428 FileSystem fs = FSUtils.getCurrentFileSystem(this.conf); 429 FsPermission defaultPerms = FSUtils.getFilePermissions(fs, this.conf, 430 HConstants.DATA_FILE_UMASK_KEY); 431 Path tmpDir = getTmpDir(conf); 432 this.hbckLockPath = new Path(tmpDir, HBCK_LOCK_FILE); 433 fs.mkdirs(tmpDir); 434 final FSDataOutputStream out = createFileWithRetries(fs, this.hbckLockPath, defaultPerms); 435 out.writeBytes(InetAddress.getLocalHost().toString()); 436 // Add a note into the file we write on why hbase2 is writing out an hbck1 lock file. 437 out.writeBytes(" Written by an hbase-2.x Master to block an " + 438 "attempt by an hbase-1.x HBCK tool making modification to state. " + 439 "See 'HBCK must match HBase server version' in the hbase refguide."); 440 out.flush(); 441 return out; 442 } catch(RemoteException e) { 443 if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){ 444 return null; 445 } else { 446 throw e; 447 } 448 } 449 } 450 451 private FSDataOutputStream createFileWithRetries(final FileSystem fs, 452 final Path hbckLockFilePath, final FsPermission defaultPerms) 453 throws IOException { 454 IOException exception = null; 455 do { 456 try { 457 return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false); 458 } catch (IOException ioe) { 459 LOG.info("Failed to create lock file " + hbckLockFilePath.getName() 460 + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of " 461 + retryCounter.getMaxAttempts()); 462 LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), 463 ioe); 464 try { 465 exception = ioe; 466 retryCounter.sleepUntilNextRetry(); 467 } catch (InterruptedException ie) { 468 throw (InterruptedIOException) new InterruptedIOException( 469 "Can't create lock file " + hbckLockFilePath.getName()) 470 .initCause(ie); 471 } 472 } 473 } while (retryCounter.shouldRetry()); 474 475 throw exception; 476 } 477 } 478 479 /** 480 * This method maintains a lock using a file. If the creation fails we return null 481 * 482 * @return FSDataOutputStream object corresponding to the newly opened lock file 483 * @throws IOException if IO failure occurs 484 */ 485 public static Pair<Path, FSDataOutputStream> checkAndMarkRunningHbck(Configuration conf, 486 RetryCounter retryCounter) throws IOException { 487 FileLockCallable callable = new FileLockCallable(conf, retryCounter); 488 ExecutorService executor = Executors.newFixedThreadPool(1); 489 FutureTask<FSDataOutputStream> futureTask = new FutureTask<>(callable); 490 executor.execute(futureTask); 491 final int timeoutInSeconds = conf.getInt( 492 "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT); 493 FSDataOutputStream stream = null; 494 try { 495 stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS); 496 } catch (ExecutionException ee) { 497 LOG.warn("Encountered exception when opening lock file", ee); 498 } catch (InterruptedException ie) { 499 LOG.warn("Interrupted when opening lock file", ie); 500 Thread.currentThread().interrupt(); 501 } catch (TimeoutException exception) { 502 // took too long to obtain lock 503 LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock"); 504 futureTask.cancel(true); 505 } finally { 506 executor.shutdownNow(); 507 } 508 return new Pair<Path, FSDataOutputStream>(callable.getHbckLockPath(), stream); 509 } 510 511 private void unlockHbck() { 512 if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) { 513 RetryCounter retryCounter = lockFileRetryCounterFactory.create(); 514 do { 515 try { 516 IOUtils.closeQuietly(hbckOutFd); 517 FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true); 518 LOG.info("Finishing hbck"); 519 return; 520 } catch (IOException ioe) { 521 LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try=" 522 + (retryCounter.getAttemptTimes() + 1) + " of " 523 + retryCounter.getMaxAttempts()); 524 LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe); 525 try { 526 retryCounter.sleepUntilNextRetry(); 527 } catch (InterruptedException ie) { 528 Thread.currentThread().interrupt(); 529 LOG.warn("Interrupted while deleting lock file" + 530 HBCK_LOCK_PATH); 531 return; 532 } 533 } 534 } while (retryCounter.shouldRetry()); 535 } 536 } 537 538 /** 539 * To repair region consistency, one must call connect() in order to repair 540 * online state. 541 */ 542 public void connect() throws IOException { 543 544 if (isExclusive()) { 545 // Grab the lock 546 Pair<Path, FSDataOutputStream> pair = 547 checkAndMarkRunningHbck(getConf(), this.lockFileRetryCounterFactory.create()); 548 HBCK_LOCK_PATH = pair.getFirst(); 549 this.hbckOutFd = pair.getSecond(); 550 if (hbckOutFd == null) { 551 setRetCode(-1); 552 LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " + 553 "[If you are sure no other instance is running, delete the lock file " + 554 HBCK_LOCK_PATH + " and rerun the tool]"); 555 throw new IOException("Duplicate hbck - Abort"); 556 } 557 558 // Make sure to cleanup the lock 559 hbckLockCleanup.set(true); 560 } 561 562 563 // Add a shutdown hook to this thread, in case user tries to 564 // kill the hbck with a ctrl-c, we want to cleanup the lock so that 565 // it is available for further calls 566 Runtime.getRuntime().addShutdownHook(new Thread() { 567 @Override 568 public void run() { 569 IOUtils.closeQuietly(HBaseFsck.this); 570 cleanupHbckZnode(); 571 unlockHbck(); 572 } 573 }); 574 575 LOG.info("Launching hbck"); 576 577 connection = (ClusterConnection)ConnectionFactory.createConnection(getConf()); 578 admin = connection.getAdmin(); 579 meta = connection.getTable(TableName.META_TABLE_NAME); 580 status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS, 581 Option.DEAD_SERVERS, Option.MASTER, Option.BACKUP_MASTERS, 582 Option.REGIONS_IN_TRANSITION, Option.HBASE_VERSION)); 583 } 584 585 /** 586 * Get deployed regions according to the region servers. 587 */ 588 private void loadDeployedRegions() throws IOException, InterruptedException { 589 // From the master, get a list of all known live region servers 590 Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet(); 591 errors.print("Number of live region servers: " + regionServers.size()); 592 if (details) { 593 for (ServerName rsinfo: regionServers) { 594 errors.print(" " + rsinfo.getServerName()); 595 } 596 } 597 598 // From the master, get a list of all dead region servers 599 Collection<ServerName> deadRegionServers = status.getDeadServerNames(); 600 errors.print("Number of dead region servers: " + deadRegionServers.size()); 601 if (details) { 602 for (ServerName name: deadRegionServers) { 603 errors.print(" " + name); 604 } 605 } 606 607 // Print the current master name and state 608 errors.print("Master: " + status.getMasterName()); 609 610 // Print the list of all backup masters 611 Collection<ServerName> backupMasters = status.getBackupMasterNames(); 612 errors.print("Number of backup masters: " + backupMasters.size()); 613 if (details) { 614 for (ServerName name: backupMasters) { 615 errors.print(" " + name); 616 } 617 } 618 619 errors.print("Average load: " + status.getAverageLoad()); 620 errors.print("Number of requests: " + status.getRequestCount()); 621 errors.print("Number of regions: " + status.getRegionCount()); 622 623 List<RegionState> rits = status.getRegionStatesInTransition(); 624 errors.print("Number of regions in transition: " + rits.size()); 625 if (details) { 626 for (RegionState state: rits) { 627 errors.print(" " + state.toDescriptiveString()); 628 } 629 } 630 631 // Determine what's deployed 632 processRegionServers(regionServers); 633 } 634 635 /** 636 * Clear the current state of hbck. 637 */ 638 private void clearState() { 639 // Make sure regionInfo is empty before starting 640 fixes = 0; 641 regionInfoMap.clear(); 642 emptyRegionInfoQualifiers.clear(); 643 tableStates.clear(); 644 errors.clear(); 645 tablesInfo.clear(); 646 orphanHdfsDirs.clear(); 647 skippedRegions.clear(); 648 } 649 650 /** 651 * This repair method analyzes hbase data in hdfs and repairs it to satisfy 652 * the table integrity rules. HBase doesn't need to be online for this 653 * operation to work. 654 */ 655 public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException { 656 // Initial pass to fix orphans. 657 if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles() 658 || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) { 659 LOG.info("Loading regioninfos HDFS"); 660 // if nothing is happening this should always complete in two iterations. 661 int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3); 662 int curIter = 0; 663 do { 664 clearState(); // clears hbck state and reset fixes to 0 and. 665 // repair what's on HDFS 666 restoreHdfsIntegrity(); 667 curIter++;// limit the number of iterations. 668 } while (fixes > 0 && curIter <= maxIterations); 669 670 // Repairs should be done in the first iteration and verification in the second. 671 // If there are more than 2 passes, something funny has happened. 672 if (curIter > 2) { 673 if (curIter == maxIterations) { 674 LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. " 675 + "Tables integrity may not be fully repaired!"); 676 } else { 677 LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations"); 678 } 679 } 680 } 681 } 682 683 /** 684 * This repair method requires the cluster to be online since it contacts 685 * region servers and the masters. It makes each region's state in HDFS, in 686 * hbase:meta, and deployments consistent. 687 * 688 * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable 689 * error. If 0, we have a clean hbase. 690 */ 691 public int onlineConsistencyRepair() throws IOException, KeeperException, 692 InterruptedException { 693 694 // get regions according to what is online on each RegionServer 695 loadDeployedRegions(); 696 // check whether hbase:meta is deployed and online 697 recordMetaRegion(); 698 // Check if hbase:meta is found only once and in the right place 699 if (!checkMetaRegion()) { 700 String errorMsg = "hbase:meta table is not consistent. "; 701 if (shouldFixAssignments()) { 702 errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state."; 703 } else { 704 errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency."; 705 } 706 errors.reportError(errorMsg + " Exiting..."); 707 return -2; 708 } 709 // Not going with further consistency check for tables when hbase:meta itself is not consistent. 710 LOG.info("Loading regionsinfo from the hbase:meta table"); 711 boolean success = loadMetaEntries(); 712 if (!success) return -1; 713 714 // Empty cells in hbase:meta? 715 reportEmptyMetaCells(); 716 717 // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta 718 if (shouldFixEmptyMetaCells()) { 719 fixEmptyMetaCells(); 720 } 721 722 // get a list of all tables that have not changed recently. 723 if (!checkMetaOnly) { 724 reportTablesInFlux(); 725 } 726 727 // Get disabled tables states 728 loadTableStates(); 729 730 // load regiondirs and regioninfos from HDFS 731 if (shouldCheckHdfs()) { 732 LOG.info("Loading region directories from HDFS"); 733 loadHdfsRegionDirs(); 734 LOG.info("Loading region information from HDFS"); 735 loadHdfsRegionInfos(); 736 } 737 738 // fix the orphan tables 739 fixOrphanTables(); 740 741 LOG.info("Checking and fixing region consistency"); 742 // Check and fix consistency 743 checkAndFixConsistency(); 744 745 // Check integrity (does not fix) 746 checkIntegrity(); 747 return errors.getErrorList().size(); 748 } 749 750 /** 751 * This method maintains an ephemeral znode. If the creation fails we return false or throw 752 * exception 753 * 754 * @return true if creating znode succeeds; false otherwise 755 * @throws IOException if IO failure occurs 756 */ 757 private boolean setMasterInMaintenanceMode() throws IOException { 758 RetryCounter retryCounter = createZNodeRetryCounterFactory.create(); 759 hbckEphemeralNodePath = ZNodePaths.joinZNode( 760 zkw.getZNodePaths().masterMaintZNode, 761 "hbck-" + Long.toString(EnvironmentEdgeManager.currentTime())); 762 do { 763 try { 764 hbckZodeCreated = ZKUtil.createEphemeralNodeAndWatch(zkw, hbckEphemeralNodePath, null); 765 if (hbckZodeCreated) { 766 break; 767 } 768 } catch (KeeperException e) { 769 if (retryCounter.getAttemptTimes() >= retryCounter.getMaxAttempts()) { 770 throw new IOException("Can't create znode " + hbckEphemeralNodePath, e); 771 } 772 // fall through and retry 773 } 774 775 LOG.warn("Fail to create znode " + hbckEphemeralNodePath + ", try=" + 776 (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts()); 777 778 try { 779 retryCounter.sleepUntilNextRetry(); 780 } catch (InterruptedException ie) { 781 throw (InterruptedIOException) new InterruptedIOException( 782 "Can't create znode " + hbckEphemeralNodePath).initCause(ie); 783 } 784 } while (retryCounter.shouldRetry()); 785 return hbckZodeCreated; 786 } 787 788 private void cleanupHbckZnode() { 789 try { 790 if (zkw != null && hbckZodeCreated) { 791 ZKUtil.deleteNode(zkw, hbckEphemeralNodePath); 792 hbckZodeCreated = false; 793 } 794 } catch (KeeperException e) { 795 // Ignore 796 if (!e.code().equals(KeeperException.Code.NONODE)) { 797 LOG.warn("Delete HBCK znode " + hbckEphemeralNodePath + " failed ", e); 798 } 799 } 800 } 801 802 /** 803 * Contacts the master and prints out cluster-wide information 804 * @return 0 on success, non-zero on failure 805 */ 806 public int onlineHbck() 807 throws IOException, KeeperException, InterruptedException, ReplicationException { 808 // print hbase server version 809 errors.print("Version: " + status.getHBaseVersion()); 810 811 // Clean start 812 clearState(); 813 // Do offline check and repair first 814 offlineHdfsIntegrityRepair(); 815 offlineReferenceFileRepair(); 816 offlineHLinkFileRepair(); 817 // If Master runs maintenance tasks (such as balancer, catalog janitor, etc) during online 818 // hbck, it is likely that hbck would be misled and report transient errors. Therefore, it 819 // is better to set Master into maintenance mode during online hbck. 820 // 821 if (!setMasterInMaintenanceMode()) { 822 LOG.warn("HBCK is running while master is not in maintenance mode, you might see transient " 823 + "error. Please run HBCK multiple times to reduce the chance of transient error."); 824 } 825 826 onlineConsistencyRepair(); 827 828 if (checkRegionBoundaries) { 829 checkRegionBoundaries(); 830 } 831 832 checkAndFixReplication(); 833 834 cleanReplicationBarrier(); 835 836 // Remove the hbck znode 837 cleanupHbckZnode(); 838 839 // Remove the hbck lock 840 unlockHbck(); 841 842 // Print table summary 843 printTableSummary(tablesInfo); 844 return errors.summarize(); 845 } 846 847 public static byte[] keyOnly(byte[] b) { 848 if (b == null) 849 return b; 850 int rowlength = Bytes.toShort(b, 0); 851 byte[] result = new byte[rowlength]; 852 System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength); 853 return result; 854 } 855 856 @Override 857 public void close() throws IOException { 858 try { 859 cleanupHbckZnode(); 860 unlockHbck(); 861 } catch (Exception io) { 862 LOG.warn(io.toString(), io); 863 } finally { 864 if (zkw != null) { 865 zkw.close(); 866 zkw = null; 867 } 868 IOUtils.closeQuietly(admin); 869 IOUtils.closeQuietly(meta); 870 IOUtils.closeQuietly(connection); 871 } 872 } 873 874 private static class RegionBoundariesInformation { 875 public byte [] regionName; 876 public byte [] metaFirstKey; 877 public byte [] metaLastKey; 878 public byte [] storesFirstKey; 879 public byte [] storesLastKey; 880 @Override 881 public String toString () { 882 return "regionName=" + Bytes.toStringBinary(regionName) + 883 "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) + 884 "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) + 885 "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) + 886 "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey); 887 } 888 } 889 890 public void checkRegionBoundaries() { 891 try { 892 ByteArrayComparator comparator = new ByteArrayComparator(); 893 List<RegionInfo> regions = MetaTableAccessor.getAllRegions(connection, true); 894 final RegionBoundariesInformation currentRegionBoundariesInformation = 895 new RegionBoundariesInformation(); 896 Path hbaseRoot = FSUtils.getRootDir(getConf()); 897 for (RegionInfo regionInfo : regions) { 898 Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable()); 899 currentRegionBoundariesInformation.regionName = regionInfo.getRegionName(); 900 // For each region, get the start and stop key from the META and compare them to the 901 // same information from the Stores. 902 Path path = new Path(tableDir, regionInfo.getEncodedName()); 903 FileSystem fs = path.getFileSystem(getConf()); 904 FileStatus[] files = fs.listStatus(path); 905 // For all the column families in this region... 906 byte[] storeFirstKey = null; 907 byte[] storeLastKey = null; 908 for (FileStatus file : files) { 909 String fileName = file.getPath().toString(); 910 fileName = fileName.substring(fileName.lastIndexOf("/") + 1); 911 if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) { 912 FileStatus[] storeFiles = fs.listStatus(file.getPath()); 913 // For all the stores in this column family. 914 for (FileStatus storeFile : storeFiles) { 915 HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), 916 CacheConfig.DISABLED, true, getConf()); 917 if ((reader.getFirstKey() != null) 918 && ((storeFirstKey == null) || (comparator.compare(storeFirstKey, 919 ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey().get()).getKey()) > 0))) { 920 storeFirstKey = ((KeyValue.KeyOnlyKeyValue)reader.getFirstKey().get()).getKey(); 921 } 922 if ((reader.getLastKey() != null) 923 && ((storeLastKey == null) || (comparator.compare(storeLastKey, 924 ((KeyValue.KeyOnlyKeyValue)reader.getLastKey().get()).getKey())) < 0)) { 925 storeLastKey = ((KeyValue.KeyOnlyKeyValue)reader.getLastKey().get()).getKey(); 926 } 927 reader.close(); 928 } 929 } 930 } 931 currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey(); 932 currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey(); 933 currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey); 934 currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey); 935 if (currentRegionBoundariesInformation.metaFirstKey.length == 0) 936 currentRegionBoundariesInformation.metaFirstKey = null; 937 if (currentRegionBoundariesInformation.metaLastKey.length == 0) 938 currentRegionBoundariesInformation.metaLastKey = null; 939 940 // For a region to be correct, we need the META start key to be smaller or equal to the 941 // smallest start key from all the stores, and the start key from the next META entry to 942 // be bigger than the last key from all the current stores. First region start key is null; 943 // Last region end key is null; some regions can be empty and not have any store. 944 945 boolean valid = true; 946 // Checking start key. 947 if ((currentRegionBoundariesInformation.storesFirstKey != null) 948 && (currentRegionBoundariesInformation.metaFirstKey != null)) { 949 valid = valid 950 && comparator.compare(currentRegionBoundariesInformation.storesFirstKey, 951 currentRegionBoundariesInformation.metaFirstKey) >= 0; 952 } 953 // Checking stop key. 954 if ((currentRegionBoundariesInformation.storesLastKey != null) 955 && (currentRegionBoundariesInformation.metaLastKey != null)) { 956 valid = valid 957 && comparator.compare(currentRegionBoundariesInformation.storesLastKey, 958 currentRegionBoundariesInformation.metaLastKey) < 0; 959 } 960 if (!valid) { 961 errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries", 962 tablesInfo.get(regionInfo.getTable())); 963 LOG.warn("Region's boundaries not aligned between stores and META for:"); 964 LOG.warn(Objects.toString(currentRegionBoundariesInformation)); 965 } 966 } 967 } catch (IOException e) { 968 LOG.error(e.toString(), e); 969 } 970 } 971 972 /** 973 * Iterates through the list of all orphan/invalid regiondirs. 974 */ 975 private void adoptHdfsOrphans(Collection<HbckRegionInfo> orphanHdfsDirs) throws IOException { 976 for (HbckRegionInfo hi : orphanHdfsDirs) { 977 LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir()); 978 adoptHdfsOrphan(hi); 979 } 980 } 981 982 /** 983 * Orphaned regions are regions without a .regioninfo file in them. We "adopt" 984 * these orphans by creating a new region, and moving the column families, 985 * recovered edits, WALs, into the new region dir. We determine the region 986 * startkey and endkeys by looking at all of the hfiles inside the column 987 * families to identify the min and max keys. The resulting region will 988 * likely violate table integrity but will be dealt with by merging 989 * overlapping regions. 990 */ 991 @SuppressWarnings("deprecation") 992 private void adoptHdfsOrphan(HbckRegionInfo hi) throws IOException { 993 Path p = hi.getHdfsRegionDir(); 994 FileSystem fs = p.getFileSystem(getConf()); 995 FileStatus[] dirs = fs.listStatus(p); 996 if (dirs == null) { 997 LOG.warn("Attempt to adopt orphan hdfs region skipped because no files present in " + 998 p + ". This dir could probably be deleted."); 999 return ; 1000 } 1001 1002 TableName tableName = hi.getTableName(); 1003 HbckTableInfo tableInfo = tablesInfo.get(tableName); 1004 Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!"); 1005 TableDescriptor template = tableInfo.getTableDescriptor(); 1006 1007 // find min and max key values 1008 Pair<byte[],byte[]> orphanRegionRange = null; 1009 for (FileStatus cf : dirs) { 1010 String cfName= cf.getPath().getName(); 1011 // TODO Figure out what the special dirs are 1012 if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue; 1013 1014 FileStatus[] hfiles = fs.listStatus(cf.getPath()); 1015 for (FileStatus hfile : hfiles) { 1016 byte[] start, end; 1017 HFile.Reader hf = null; 1018 try { 1019 hf = HFile.createReader(fs, hfile.getPath(), CacheConfig.DISABLED, true, getConf()); 1020 hf.loadFileInfo(); 1021 Optional<Cell> startKv = hf.getFirstKey(); 1022 start = CellUtil.cloneRow(startKv.get()); 1023 Optional<Cell> endKv = hf.getLastKey(); 1024 end = CellUtil.cloneRow(endKv.get()); 1025 } catch (IOException ioe) { 1026 LOG.warn("Problem reading orphan file " + hfile + ", skipping"); 1027 continue; 1028 } catch (NullPointerException ioe) { 1029 LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping"); 1030 continue; 1031 } finally { 1032 if (hf != null) { 1033 hf.close(); 1034 } 1035 } 1036 1037 // expand the range to include the range of all hfiles 1038 if (orphanRegionRange == null) { 1039 // first range 1040 orphanRegionRange = new Pair<>(start, end); 1041 } else { 1042 // TODO add test 1043 1044 // expand range only if the hfile is wider. 1045 if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) { 1046 orphanRegionRange.setFirst(start); 1047 } 1048 if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) { 1049 orphanRegionRange.setSecond(end); 1050 } 1051 } 1052 } 1053 } 1054 if (orphanRegionRange == null) { 1055 LOG.warn("No data in dir " + p + ", sidelining data"); 1056 fixes++; 1057 sidelineRegionDir(fs, hi); 1058 return; 1059 } 1060 LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " + 1061 Bytes.toString(orphanRegionRange.getSecond()) + ")"); 1062 1063 // create new region on hdfs. move data into place. 1064 RegionInfo regionInfo = RegionInfoBuilder.newBuilder(template.getTableName()) 1065 .setStartKey(orphanRegionRange.getFirst()) 1066 .setEndKey(Bytes.add(orphanRegionRange.getSecond(), new byte[1])) 1067 .build(); 1068 LOG.info("Creating new region : " + regionInfo); 1069 HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), regionInfo, template); 1070 Path target = region.getRegionFileSystem().getRegionDir(); 1071 1072 // rename all the data to new region 1073 mergeRegionDirs(target, hi); 1074 fixes++; 1075 } 1076 1077 /** 1078 * This method determines if there are table integrity errors in HDFS. If 1079 * there are errors and the appropriate "fix" options are enabled, the method 1080 * will first correct orphan regions making them into legit regiondirs, and 1081 * then reload to merge potentially overlapping regions. 1082 * 1083 * @return number of table integrity errors found 1084 */ 1085 private int restoreHdfsIntegrity() throws IOException, InterruptedException { 1086 // Determine what's on HDFS 1087 LOG.info("Loading HBase regioninfo from HDFS..."); 1088 loadHdfsRegionDirs(); // populating regioninfo table. 1089 1090 int errs = errors.getErrorList().size(); 1091 // First time just get suggestions. 1092 tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs. 1093 checkHdfsIntegrity(false, false); 1094 1095 if (errors.getErrorList().size() == errs) { 1096 LOG.info("No integrity errors. We are done with this phase. Glorious."); 1097 return 0; 1098 } 1099 1100 if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) { 1101 adoptHdfsOrphans(orphanHdfsDirs); 1102 // TODO optimize by incrementally adding instead of reloading. 1103 } 1104 1105 // Make sure there are no holes now. 1106 if (shouldFixHdfsHoles()) { 1107 clearState(); // this also resets # fixes. 1108 loadHdfsRegionDirs(); 1109 tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs. 1110 tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false); 1111 } 1112 1113 // Now we fix overlaps 1114 if (shouldFixHdfsOverlaps()) { 1115 // second pass we fix overlaps. 1116 clearState(); // this also resets # fixes. 1117 loadHdfsRegionDirs(); 1118 tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs. 1119 tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps()); 1120 } 1121 1122 return errors.getErrorList().size(); 1123 } 1124 1125 /** 1126 * Scan all the store file names to find any lingering reference files, 1127 * which refer to some none-exiting files. If "fix" option is enabled, 1128 * any lingering reference file will be sidelined if found. 1129 * <p> 1130 * Lingering reference file prevents a region from opening. It has to 1131 * be fixed before a cluster can start properly. 1132 */ 1133 private void offlineReferenceFileRepair() throws IOException, InterruptedException { 1134 clearState(); 1135 Configuration conf = getConf(); 1136 Path hbaseRoot = FSUtils.getRootDir(conf); 1137 FileSystem fs = hbaseRoot.getFileSystem(conf); 1138 LOG.info("Computing mapping of all store files"); 1139 Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, 1140 new FSUtils.ReferenceFileFilter(fs), executor, errors); 1141 errors.print(""); 1142 LOG.info("Validating mapping using HDFS state"); 1143 for (Path path: allFiles.values()) { 1144 Path referredToFile = StoreFileInfo.getReferredToFile(path); 1145 if (fs.exists(referredToFile)) continue; // good, expected 1146 1147 // Found a lingering reference file 1148 errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE, 1149 "Found lingering reference file " + path); 1150 if (!shouldFixReferenceFiles()) continue; 1151 1152 // Now, trying to fix it since requested 1153 boolean success = false; 1154 String pathStr = path.toString(); 1155 1156 // A reference file path should be like 1157 // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name 1158 // Up 5 directories to get the root folder. 1159 // So the file will be sidelined to a similar folder structure. 1160 int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR); 1161 for (int i = 0; index > 0 && i < 5; i++) { 1162 index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1); 1163 } 1164 if (index > 0) { 1165 Path rootDir = getSidelineDir(); 1166 Path dst = new Path(rootDir, pathStr.substring(index + 1)); 1167 fs.mkdirs(dst.getParent()); 1168 LOG.info("Trying to sideline reference file " 1169 + path + " to " + dst); 1170 setShouldRerun(); 1171 1172 success = fs.rename(path, dst); 1173 debugLsr(dst); 1174 1175 } 1176 if (!success) { 1177 LOG.error("Failed to sideline reference file " + path); 1178 } 1179 } 1180 } 1181 1182 /** 1183 * Scan all the store file names to find any lingering HFileLink files, 1184 * which refer to some none-exiting files. If "fix" option is enabled, 1185 * any lingering HFileLink file will be sidelined if found. 1186 */ 1187 private void offlineHLinkFileRepair() throws IOException, InterruptedException { 1188 Configuration conf = getConf(); 1189 Path hbaseRoot = FSUtils.getRootDir(conf); 1190 FileSystem fs = hbaseRoot.getFileSystem(conf); 1191 LOG.info("Computing mapping of all link files"); 1192 Map<String, Path> allFiles = FSUtils 1193 .getTableStoreFilePathMap(fs, hbaseRoot, new FSUtils.HFileLinkFilter(), executor, errors); 1194 errors.print(""); 1195 1196 LOG.info("Validating mapping using HDFS state"); 1197 for (Path path : allFiles.values()) { 1198 // building HFileLink object to gather locations 1199 HFileLink actualLink = HFileLink.buildFromHFileLinkPattern(conf, path); 1200 if (actualLink.exists(fs)) continue; // good, expected 1201 1202 // Found a lingering HFileLink 1203 errors.reportError(ERROR_CODE.LINGERING_HFILELINK, "Found lingering HFileLink " + path); 1204 if (!shouldFixHFileLinks()) continue; 1205 1206 // Now, trying to fix it since requested 1207 setShouldRerun(); 1208 1209 // An HFileLink path should be like 1210 // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/linkedtable=linkedregionname-linkedhfilename 1211 // sidelineing will happen in the ${hbase.rootdir}/${sidelinedir} directory with the same folder structure. 1212 boolean success = sidelineFile(fs, hbaseRoot, path); 1213 1214 if (!success) { 1215 LOG.error("Failed to sideline HFileLink file " + path); 1216 } 1217 1218 // An HFileLink backreference path should be like 1219 // ${hbase.rootdir}/archive/data/namespace/table_name/region_id/family_name/.links-linkedhfilename 1220 // sidelineing will happen in the ${hbase.rootdir}/${sidelinedir} directory with the same folder structure. 1221 Path backRefPath = FileLink.getBackReferencesDir(HFileArchiveUtil 1222 .getStoreArchivePath(conf, HFileLink.getReferencedTableName(path.getName().toString()), 1223 HFileLink.getReferencedRegionName(path.getName().toString()), 1224 path.getParent().getName()), 1225 HFileLink.getReferencedHFileName(path.getName().toString())); 1226 success = sidelineFile(fs, hbaseRoot, backRefPath); 1227 1228 if (!success) { 1229 LOG.error("Failed to sideline HFileLink backreference file " + path); 1230 } 1231 } 1232 } 1233 1234 private boolean sidelineFile(FileSystem fs, Path hbaseRoot, Path path) throws IOException { 1235 URI uri = hbaseRoot.toUri().relativize(path.toUri()); 1236 if (uri.isAbsolute()) return false; 1237 String relativePath = uri.getPath(); 1238 Path rootDir = getSidelineDir(); 1239 Path dst = new Path(rootDir, relativePath); 1240 boolean pathCreated = fs.mkdirs(dst.getParent()); 1241 if (!pathCreated) { 1242 LOG.error("Failed to create path: " + dst.getParent()); 1243 return false; 1244 } 1245 LOG.info("Trying to sideline file " + path + " to " + dst); 1246 return fs.rename(path, dst); 1247 } 1248 1249 /** 1250 * TODO -- need to add tests for this. 1251 */ 1252 private void reportEmptyMetaCells() { 1253 errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " + 1254 emptyRegionInfoQualifiers.size()); 1255 if (details) { 1256 for (Result r: emptyRegionInfoQualifiers) { 1257 errors.print(" " + r); 1258 } 1259 } 1260 } 1261 1262 /** 1263 * TODO -- need to add tests for this. 1264 */ 1265 private void reportTablesInFlux() { 1266 AtomicInteger numSkipped = new AtomicInteger(0); 1267 TableDescriptor[] allTables = getTables(numSkipped); 1268 errors.print("Number of Tables: " + allTables.length); 1269 if (details) { 1270 if (numSkipped.get() > 0) { 1271 errors.detail("Number of Tables in flux: " + numSkipped.get()); 1272 } 1273 for (TableDescriptor td : allTables) { 1274 errors.detail(" Table: " + td.getTableName() + "\t" + 1275 (td.isReadOnly() ? "ro" : "rw") + "\t" + 1276 (td.isMetaRegion() ? "META" : " ") + "\t" + 1277 " families: " + td.getColumnFamilyCount()); 1278 } 1279 } 1280 } 1281 1282 public HbckErrorReporter getErrors() { 1283 return errors; 1284 } 1285 1286 /** 1287 * Populate hbi's from regionInfos loaded from file system. 1288 */ 1289 private SortedMap<TableName, HbckTableInfo> loadHdfsRegionInfos() 1290 throws IOException, InterruptedException { 1291 tablesInfo.clear(); // regenerating the data 1292 // generate region split structure 1293 Collection<HbckRegionInfo> hbckRegionInfos = regionInfoMap.values(); 1294 1295 // Parallelized read of .regioninfo files. 1296 List<WorkItemHdfsRegionInfo> hbis = new ArrayList<>(hbckRegionInfos.size()); 1297 List<Future<Void>> hbiFutures; 1298 1299 for (HbckRegionInfo hbi : hbckRegionInfos) { 1300 WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors); 1301 hbis.add(work); 1302 } 1303 1304 // Submit and wait for completion 1305 hbiFutures = executor.invokeAll(hbis); 1306 1307 for(int i=0; i<hbiFutures.size(); i++) { 1308 WorkItemHdfsRegionInfo work = hbis.get(i); 1309 Future<Void> f = hbiFutures.get(i); 1310 try { 1311 f.get(); 1312 } catch(ExecutionException e) { 1313 LOG.warn("Failed to read .regioninfo file for region " + 1314 work.hbi.getRegionNameAsString(), e.getCause()); 1315 } 1316 } 1317 1318 Path hbaseRoot = FSUtils.getRootDir(getConf()); 1319 FileSystem fs = hbaseRoot.getFileSystem(getConf()); 1320 // serialized table info gathering. 1321 for (HbckRegionInfo hbi: hbckRegionInfos) { 1322 1323 if (hbi.getHdfsHRI() == null) { 1324 // was an orphan 1325 continue; 1326 } 1327 1328 1329 // get table name from hdfs, populate various HBaseFsck tables. 1330 TableName tableName = hbi.getTableName(); 1331 if (tableName == null) { 1332 // There was an entry in hbase:meta not in the HDFS? 1333 LOG.warn("tableName was null for: " + hbi); 1334 continue; 1335 } 1336 1337 HbckTableInfo modTInfo = tablesInfo.get(tableName); 1338 if (modTInfo == null) { 1339 // only executed once per table. 1340 modTInfo = new HbckTableInfo(tableName, this); 1341 tablesInfo.put(tableName, modTInfo); 1342 try { 1343 TableDescriptor htd = 1344 FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName); 1345 modTInfo.htds.add(htd); 1346 } catch (IOException ioe) { 1347 if (!orphanTableDirs.containsKey(tableName)) { 1348 LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe); 1349 //should only report once for each table 1350 errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE, 1351 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName); 1352 Set<String> columns = new HashSet<>(); 1353 orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi)); 1354 } 1355 } 1356 } 1357 if (!hbi.isSkipChecks()) { 1358 modTInfo.addRegionInfo(hbi); 1359 } 1360 } 1361 1362 loadTableInfosForTablesWithNoRegion(); 1363 errors.print(""); 1364 1365 return tablesInfo; 1366 } 1367 1368 /** 1369 * To get the column family list according to the column family dirs 1370 * @param columns 1371 * @param hbi 1372 * @return a set of column families 1373 * @throws IOException 1374 */ 1375 private Set<String> getColumnFamilyList(Set<String> columns, HbckRegionInfo hbi) 1376 throws IOException { 1377 Path regionDir = hbi.getHdfsRegionDir(); 1378 FileSystem fs = regionDir.getFileSystem(getConf()); 1379 FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs)); 1380 for (FileStatus subdir : subDirs) { 1381 String columnfamily = subdir.getPath().getName(); 1382 columns.add(columnfamily); 1383 } 1384 return columns; 1385 } 1386 1387 /** 1388 * To fabricate a .tableinfo file with following contents<br> 1389 * 1. the correct tablename <br> 1390 * 2. the correct colfamily list<br> 1391 * 3. the default properties for both {@link TableDescriptor} and {@link ColumnFamilyDescriptor}<br> 1392 * @throws IOException 1393 */ 1394 private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName, 1395 Set<String> columns) throws IOException { 1396 if (columns ==null || columns.isEmpty()) return false; 1397 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); 1398 for (String columnfamimly : columns) { 1399 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(columnfamimly)); 1400 } 1401 fstd.createTableDescriptor(builder.build(), true); 1402 return true; 1403 } 1404 1405 /** 1406 * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br> 1407 * @throws IOException 1408 */ 1409 public void fixEmptyMetaCells() throws IOException { 1410 if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) { 1411 LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows."); 1412 for (Result region : emptyRegionInfoQualifiers) { 1413 deleteMetaRegion(region.getRow()); 1414 errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL); 1415 } 1416 emptyRegionInfoQualifiers.clear(); 1417 } 1418 } 1419 1420 /** 1421 * To fix orphan table by creating a .tableinfo file under tableDir <br> 1422 * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br> 1423 * 2. else create a default .tableinfo file with following items<br> 1424 * 2.1 the correct tablename <br> 1425 * 2.2 the correct colfamily list<br> 1426 * 2.3 the default properties for both {@link TableDescriptor} and {@link ColumnFamilyDescriptor}<br> 1427 * @throws IOException 1428 */ 1429 public void fixOrphanTables() throws IOException { 1430 if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) { 1431 1432 List<TableName> tmpList = new ArrayList<>(orphanTableDirs.keySet().size()); 1433 tmpList.addAll(orphanTableDirs.keySet()); 1434 TableDescriptor[] htds = getTableDescriptors(tmpList); 1435 Iterator<Entry<TableName, Set<String>>> iter = 1436 orphanTableDirs.entrySet().iterator(); 1437 int j = 0; 1438 int numFailedCase = 0; 1439 FSTableDescriptors fstd = new FSTableDescriptors(getConf()); 1440 while (iter.hasNext()) { 1441 Entry<TableName, Set<String>> entry = 1442 iter.next(); 1443 TableName tableName = entry.getKey(); 1444 LOG.info("Trying to fix orphan table error: " + tableName); 1445 if (j < htds.length) { 1446 if (tableName.equals(htds[j].getTableName())) { 1447 TableDescriptor htd = htds[j]; 1448 LOG.info("fixing orphan table: " + tableName + " from cache"); 1449 fstd.createTableDescriptor(htd, true); 1450 j++; 1451 iter.remove(); 1452 } 1453 } else { 1454 if (fabricateTableInfo(fstd, tableName, entry.getValue())) { 1455 LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file"); 1456 LOG.warn("Strongly recommend to modify the TableDescriptor if necessary for: " + tableName); 1457 iter.remove(); 1458 } else { 1459 LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information"); 1460 numFailedCase++; 1461 } 1462 } 1463 fixes++; 1464 } 1465 1466 if (orphanTableDirs.isEmpty()) { 1467 // all orphanTableDirs are luckily recovered 1468 // re-run doFsck after recovering the .tableinfo file 1469 setShouldRerun(); 1470 LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed"); 1471 } else if (numFailedCase > 0) { 1472 LOG.error("Failed to fix " + numFailedCase 1473 + " OrphanTables with default .tableinfo files"); 1474 } 1475 1476 } 1477 //cleanup the list 1478 orphanTableDirs.clear(); 1479 1480 } 1481 1482 /** 1483 * Log an appropriate message about whether or not overlapping merges are computed in parallel. 1484 */ 1485 private void logParallelMerge() { 1486 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) { 1487 LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" + 1488 " false to run serially."); 1489 } else { 1490 LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to" + 1491 " true to run in parallel."); 1492 } 1493 } 1494 1495 private SortedMap<TableName, HbckTableInfo> checkHdfsIntegrity(boolean fixHoles, 1496 boolean fixOverlaps) throws IOException { 1497 LOG.info("Checking HBase region split map from HDFS data..."); 1498 logParallelMerge(); 1499 for (HbckTableInfo tInfo : tablesInfo.values()) { 1500 TableIntegrityErrorHandler handler; 1501 if (fixHoles || fixOverlaps) { 1502 handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(), 1503 fixHoles, fixOverlaps); 1504 } else { 1505 handler = tInfo.new IntegrityFixSuggester(tInfo, errors); 1506 } 1507 if (!tInfo.checkRegionChain(handler)) { 1508 // should dump info as well. 1509 errors.report("Found inconsistency in table " + tInfo.getName()); 1510 } 1511 } 1512 return tablesInfo; 1513 } 1514 1515 Path getSidelineDir() throws IOException { 1516 if (sidelineDir == null) { 1517 Path hbaseDir = FSUtils.getRootDir(getConf()); 1518 Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME); 1519 sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-" 1520 + startMillis); 1521 } 1522 return sidelineDir; 1523 } 1524 1525 /** 1526 * Sideline a region dir (instead of deleting it) 1527 */ 1528 Path sidelineRegionDir(FileSystem fs, HbckRegionInfo hi) throws IOException { 1529 return sidelineRegionDir(fs, null, hi); 1530 } 1531 1532 /** 1533 * Sideline a region dir (instead of deleting it) 1534 * 1535 * @param parentDir if specified, the region will be sidelined to folder like 1536 * {@literal .../parentDir/<table name>/<region name>}. The purpose is to group together 1537 * similar regions sidelined, for example, those regions should be bulk loaded back later 1538 * on. If NULL, it is ignored. 1539 */ 1540 Path sidelineRegionDir(FileSystem fs, 1541 String parentDir, HbckRegionInfo hi) throws IOException { 1542 TableName tableName = hi.getTableName(); 1543 Path regionDir = hi.getHdfsRegionDir(); 1544 1545 if (!fs.exists(regionDir)) { 1546 LOG.warn("No previous " + regionDir + " exists. Continuing."); 1547 return null; 1548 } 1549 1550 Path rootDir = getSidelineDir(); 1551 if (parentDir != null) { 1552 rootDir = new Path(rootDir, parentDir); 1553 } 1554 Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName); 1555 Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName()); 1556 fs.mkdirs(sidelineRegionDir); 1557 boolean success = false; 1558 FileStatus[] cfs = fs.listStatus(regionDir); 1559 if (cfs == null) { 1560 LOG.info("Region dir is empty: " + regionDir); 1561 } else { 1562 for (FileStatus cf : cfs) { 1563 Path src = cf.getPath(); 1564 Path dst = new Path(sidelineRegionDir, src.getName()); 1565 if (fs.isFile(src)) { 1566 // simple file 1567 success = fs.rename(src, dst); 1568 if (!success) { 1569 String msg = "Unable to rename file " + src + " to " + dst; 1570 LOG.error(msg); 1571 throw new IOException(msg); 1572 } 1573 continue; 1574 } 1575 1576 // is a directory. 1577 fs.mkdirs(dst); 1578 1579 LOG.info("Sidelining files from " + src + " into containing region " + dst); 1580 // FileSystem.rename is inconsistent with directories -- if the 1581 // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir, 1582 // it moves the src into the dst dir resulting in (foo/a/b). If 1583 // the dst does not exist, and the src a dir, src becomes dst. (foo/b) 1584 FileStatus[] hfiles = fs.listStatus(src); 1585 if (hfiles != null && hfiles.length > 0) { 1586 for (FileStatus hfile : hfiles) { 1587 success = fs.rename(hfile.getPath(), dst); 1588 if (!success) { 1589 String msg = "Unable to rename file " + src + " to " + dst; 1590 LOG.error(msg); 1591 throw new IOException(msg); 1592 } 1593 } 1594 } 1595 LOG.debug("Sideline directory contents:"); 1596 debugLsr(sidelineRegionDir); 1597 } 1598 } 1599 1600 LOG.info("Removing old region dir: " + regionDir); 1601 success = fs.delete(regionDir, true); 1602 if (!success) { 1603 String msg = "Unable to delete dir " + regionDir; 1604 LOG.error(msg); 1605 throw new IOException(msg); 1606 } 1607 return sidelineRegionDir; 1608 } 1609 1610 /** 1611 * Load the list of disabled tables in ZK into local set. 1612 * @throws ZooKeeperConnectionException 1613 * @throws IOException 1614 */ 1615 private void loadTableStates() 1616 throws IOException { 1617 tableStates = MetaTableAccessor.getTableStates(connection); 1618 // Add hbase:meta so this tool keeps working. In hbase2, meta is always enabled though it 1619 // has no entry in the table states. HBCK doesn't work right w/ hbase2 but just do this in 1620 // meantime. 1621 this.tableStates.put(TableName.META_TABLE_NAME, 1622 new TableState(TableName.META_TABLE_NAME, TableState.State.ENABLED)); 1623 } 1624 1625 /** 1626 * Check if the specified region's table is disabled. 1627 * @param tableName table to check status of 1628 */ 1629 boolean isTableDisabled(TableName tableName) { 1630 return tableStates.containsKey(tableName) 1631 && tableStates.get(tableName) 1632 .inStates(TableState.State.DISABLED, TableState.State.DISABLING); 1633 } 1634 1635 /** 1636 * Scan HDFS for all regions, recording their information into 1637 * regionInfoMap 1638 */ 1639 public void loadHdfsRegionDirs() throws IOException, InterruptedException { 1640 Path rootDir = FSUtils.getRootDir(getConf()); 1641 FileSystem fs = rootDir.getFileSystem(getConf()); 1642 1643 // list all tables from HDFS 1644 List<FileStatus> tableDirs = Lists.newArrayList(); 1645 1646 boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME)); 1647 1648 List<Path> paths = FSUtils.getTableDirs(fs, rootDir); 1649 for (Path path : paths) { 1650 TableName tableName = FSUtils.getTableName(path); 1651 if ((!checkMetaOnly && 1652 isTableIncluded(tableName)) || 1653 tableName.equals(TableName.META_TABLE_NAME)) { 1654 tableDirs.add(fs.getFileStatus(path)); 1655 } 1656 } 1657 1658 // verify that version file exists 1659 if (!foundVersionFile) { 1660 errors.reportError(ERROR_CODE.NO_VERSION_FILE, 1661 "Version file does not exist in root dir " + rootDir); 1662 if (shouldFixVersionFile()) { 1663 LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME 1664 + " file."); 1665 setShouldRerun(); 1666 FSUtils.setVersion(fs, rootDir, getConf().getInt( 1667 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt( 1668 HConstants.VERSION_FILE_WRITE_ATTEMPTS, 1669 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS)); 1670 } 1671 } 1672 1673 // Avoid multithreading at table-level because already multithreaded internally at 1674 // region-level. Additionally multithreading at table-level can lead to deadlock 1675 // if there are many tables in the cluster. Since there are a limited # of threads 1676 // in the executor's thread pool and if we multithread at the table-level by putting 1677 // WorkItemHdfsDir callables into the executor, then we will have some threads in the 1678 // executor tied up solely in waiting for the tables' region-level calls to complete. 1679 // If there are enough tables then there will be no actual threads in the pool left 1680 // for the region-level callables to be serviced. 1681 for (FileStatus tableDir : tableDirs) { 1682 LOG.debug("Loading region dirs from " +tableDir.getPath()); 1683 WorkItemHdfsDir item = new WorkItemHdfsDir(fs, errors, tableDir); 1684 try { 1685 item.call(); 1686 } catch (ExecutionException e) { 1687 LOG.warn("Could not completely load table dir " + 1688 tableDir.getPath(), e.getCause()); 1689 } 1690 } 1691 errors.print(""); 1692 } 1693 1694 /** 1695 * Record the location of the hbase:meta region as found in ZooKeeper. 1696 */ 1697 private boolean recordMetaRegion() throws IOException { 1698 RegionLocations rl = connection.locateRegion(TableName.META_TABLE_NAME, 1699 HConstants.EMPTY_START_ROW, false, false); 1700 if (rl == null) { 1701 errors.reportError(ERROR_CODE.NULL_META_REGION, 1702 "META region was not found in ZooKeeper"); 1703 return false; 1704 } 1705 for (HRegionLocation metaLocation : rl.getRegionLocations()) { 1706 // Check if Meta region is valid and existing 1707 if (metaLocation == null ) { 1708 errors.reportError(ERROR_CODE.NULL_META_REGION, 1709 "META region location is null"); 1710 return false; 1711 } 1712 if (metaLocation.getRegionInfo() == null) { 1713 errors.reportError(ERROR_CODE.NULL_META_REGION, 1714 "META location regionInfo is null"); 1715 return false; 1716 } 1717 if (metaLocation.getHostname() == null) { 1718 errors.reportError(ERROR_CODE.NULL_META_REGION, 1719 "META location hostName is null"); 1720 return false; 1721 } 1722 ServerName sn = metaLocation.getServerName(); 1723 HbckRegionInfo.MetaEntry m = new HbckRegionInfo.MetaEntry(metaLocation.getRegion(), sn, 1724 EnvironmentEdgeManager.currentTime()); 1725 HbckRegionInfo hbckRegionInfo = regionInfoMap.get(metaLocation.getRegion().getEncodedName()); 1726 if (hbckRegionInfo == null) { 1727 regionInfoMap.put(metaLocation.getRegion().getEncodedName(), new HbckRegionInfo(m)); 1728 } else { 1729 hbckRegionInfo.setMetaEntry(m); 1730 } 1731 } 1732 return true; 1733 } 1734 1735 private ZKWatcher createZooKeeperWatcher() throws IOException { 1736 return new ZKWatcher(getConf(), "hbase Fsck", new Abortable() { 1737 @Override 1738 public void abort(String why, Throwable e) { 1739 LOG.error(why, e); 1740 System.exit(1); 1741 } 1742 1743 @Override 1744 public boolean isAborted() { 1745 return false; 1746 } 1747 1748 }); 1749 } 1750 1751 private ServerName getMetaRegionServerName(int replicaId) 1752 throws IOException, KeeperException { 1753 return new MetaTableLocator().getMetaRegionLocation(zkw, replicaId); 1754 } 1755 1756 /** 1757 * Contacts each regionserver and fetches metadata about regions. 1758 * @param regionServerList - the list of region servers to connect to 1759 * @throws IOException if a remote or network exception occurs 1760 */ 1761 void processRegionServers(Collection<ServerName> regionServerList) 1762 throws IOException, InterruptedException { 1763 1764 List<WorkItemRegion> workItems = new ArrayList<>(regionServerList.size()); 1765 List<Future<Void>> workFutures; 1766 1767 // loop to contact each region server in parallel 1768 for (ServerName rsinfo: regionServerList) { 1769 workItems.add(new WorkItemRegion(this, rsinfo, errors, connection)); 1770 } 1771 1772 workFutures = executor.invokeAll(workItems); 1773 1774 for(int i=0; i<workFutures.size(); i++) { 1775 WorkItemRegion item = workItems.get(i); 1776 Future<Void> f = workFutures.get(i); 1777 try { 1778 f.get(); 1779 } catch(ExecutionException e) { 1780 LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(), 1781 e.getCause()); 1782 } 1783 } 1784 } 1785 1786 /** 1787 * Check consistency of all regions that have been found in previous phases. 1788 */ 1789 private void checkAndFixConsistency() 1790 throws IOException, KeeperException, InterruptedException { 1791 // Divide the checks in two phases. One for default/primary replicas and another 1792 // for the non-primary ones. Keeps code cleaner this way. 1793 1794 List<CheckRegionConsistencyWorkItem> workItems = new ArrayList<>(regionInfoMap.size()); 1795 for (java.util.Map.Entry<String, HbckRegionInfo> e: regionInfoMap.entrySet()) { 1796 if (e.getValue().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { 1797 workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue())); 1798 } 1799 } 1800 checkRegionConsistencyConcurrently(workItems); 1801 1802 boolean prevHdfsCheck = shouldCheckHdfs(); 1803 setCheckHdfs(false); //replicas don't have any hdfs data 1804 // Run a pass over the replicas and fix any assignment issues that exist on the currently 1805 // deployed/undeployed replicas. 1806 List<CheckRegionConsistencyWorkItem> replicaWorkItems = new ArrayList<>(regionInfoMap.size()); 1807 for (java.util.Map.Entry<String, HbckRegionInfo> e: regionInfoMap.entrySet()) { 1808 if (e.getValue().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 1809 replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue())); 1810 } 1811 } 1812 checkRegionConsistencyConcurrently(replicaWorkItems); 1813 setCheckHdfs(prevHdfsCheck); 1814 1815 // If some regions is skipped during checkRegionConsistencyConcurrently() phase, we might 1816 // not get accurate state of the hbase if continuing. The config here allows users to tune 1817 // the tolerance of number of skipped region. 1818 // TODO: evaluate the consequence to continue the hbck operation without config. 1819 int terminateThreshold = getConf().getInt("hbase.hbck.skipped.regions.limit", 0); 1820 int numOfSkippedRegions = skippedRegions.size(); 1821 if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) { 1822 throw new IOException(numOfSkippedRegions 1823 + " region(s) could not be checked or repaired. See logs for detail."); 1824 } 1825 1826 if (shouldCheckHdfs()) { 1827 checkAndFixTableStates(); 1828 } 1829 } 1830 1831 /** 1832 * Check consistency of all regions using mulitple threads concurrently. 1833 */ 1834 private void checkRegionConsistencyConcurrently( 1835 final List<CheckRegionConsistencyWorkItem> workItems) 1836 throws IOException, KeeperException, InterruptedException { 1837 if (workItems.isEmpty()) { 1838 return; // nothing to check 1839 } 1840 1841 List<Future<Void>> workFutures = executor.invokeAll(workItems); 1842 for(Future<Void> f: workFutures) { 1843 try { 1844 f.get(); 1845 } catch(ExecutionException e1) { 1846 LOG.warn("Could not check region consistency " , e1.getCause()); 1847 if (e1.getCause() instanceof IOException) { 1848 throw (IOException)e1.getCause(); 1849 } else if (e1.getCause() instanceof KeeperException) { 1850 throw (KeeperException)e1.getCause(); 1851 } else if (e1.getCause() instanceof InterruptedException) { 1852 throw (InterruptedException)e1.getCause(); 1853 } else { 1854 throw new IOException(e1.getCause()); 1855 } 1856 } 1857 } 1858 } 1859 1860 class CheckRegionConsistencyWorkItem implements Callable<Void> { 1861 private final String key; 1862 private final HbckRegionInfo hbi; 1863 1864 CheckRegionConsistencyWorkItem(String key, HbckRegionInfo hbi) { 1865 this.key = key; 1866 this.hbi = hbi; 1867 } 1868 1869 @Override 1870 public synchronized Void call() throws Exception { 1871 try { 1872 checkRegionConsistency(key, hbi); 1873 } catch (Exception e) { 1874 // If the region is non-META region, skip this region and send warning/error message; if 1875 // the region is META region, we should not continue. 1876 LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString() 1877 + "'.", e); 1878 if (hbi.getHdfsHRI().isMetaRegion()) { 1879 throw e; 1880 } 1881 LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'"); 1882 addSkippedRegion(hbi); 1883 } 1884 return null; 1885 } 1886 } 1887 1888 private void addSkippedRegion(final HbckRegionInfo hbi) { 1889 Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName()); 1890 if (skippedRegionNames == null) { 1891 skippedRegionNames = new HashSet<>(); 1892 } 1893 skippedRegionNames.add(hbi.getRegionNameAsString()); 1894 skippedRegions.put(hbi.getTableName(), skippedRegionNames); 1895 } 1896 1897 /** 1898 * Check and fix table states, assumes full info available: 1899 * - tableInfos 1900 * - empty tables loaded 1901 */ 1902 private void checkAndFixTableStates() throws IOException { 1903 // first check dangling states 1904 for (Entry<TableName, TableState> entry : tableStates.entrySet()) { 1905 TableName tableName = entry.getKey(); 1906 TableState tableState = entry.getValue(); 1907 HbckTableInfo tableInfo = tablesInfo.get(tableName); 1908 if (isTableIncluded(tableName) 1909 && !tableName.isSystemTable() 1910 && tableInfo == null) { 1911 if (fixMeta) { 1912 MetaTableAccessor.deleteTableState(connection, tableName); 1913 TableState state = MetaTableAccessor.getTableState(connection, tableName); 1914 if (state != null) { 1915 errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE, 1916 tableName + " unable to delete dangling table state " + tableState); 1917 } 1918 } else if (!checkMetaOnly) { 1919 // dangling table state in meta if checkMetaOnly is false. If checkMetaOnly is 1920 // true, tableInfo will be null as tablesInfo are not polulated for all tables from hdfs 1921 errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE, 1922 tableName + " has dangling table state " + tableState); 1923 } 1924 } 1925 } 1926 // check that all tables have states 1927 for (TableName tableName : tablesInfo.keySet()) { 1928 if (isTableIncluded(tableName) && !tableStates.containsKey(tableName)) { 1929 if (fixMeta) { 1930 MetaTableAccessor.updateTableState(connection, tableName, TableState.State.ENABLED); 1931 TableState newState = MetaTableAccessor.getTableState(connection, tableName); 1932 if (newState == null) { 1933 errors.reportError(ERROR_CODE.NO_TABLE_STATE, 1934 "Unable to change state for table " + tableName + " in meta "); 1935 } 1936 } else { 1937 errors.reportError(ERROR_CODE.NO_TABLE_STATE, 1938 tableName + " has no state in meta "); 1939 } 1940 } 1941 } 1942 } 1943 1944 private void preCheckPermission() throws IOException, AccessDeniedException { 1945 if (shouldIgnorePreCheckPermission()) { 1946 return; 1947 } 1948 1949 Path hbaseDir = FSUtils.getRootDir(getConf()); 1950 FileSystem fs = hbaseDir.getFileSystem(getConf()); 1951 UserProvider userProvider = UserProvider.instantiate(getConf()); 1952 UserGroupInformation ugi = userProvider.getCurrent().getUGI(); 1953 FileStatus[] files = fs.listStatus(hbaseDir); 1954 for (FileStatus file : files) { 1955 try { 1956 FSUtils.checkAccess(ugi, file, FsAction.WRITE); 1957 } catch (AccessDeniedException ace) { 1958 LOG.warn("Got AccessDeniedException when preCheckPermission ", ace); 1959 errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName() 1960 + " does not have write perms to " + file.getPath() 1961 + ". Please rerun hbck as hdfs user " + file.getOwner()); 1962 throw ace; 1963 } 1964 } 1965 } 1966 1967 /** 1968 * Deletes region from meta table 1969 */ 1970 private void deleteMetaRegion(HbckRegionInfo hi) throws IOException { 1971 deleteMetaRegion(hi.getMetaEntry().getRegionName()); 1972 } 1973 1974 /** 1975 * Deletes region from meta table 1976 */ 1977 private void deleteMetaRegion(byte[] metaKey) throws IOException { 1978 Delete d = new Delete(metaKey); 1979 meta.delete(d); 1980 LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" ); 1981 } 1982 1983 /** 1984 * Reset the split parent region info in meta table 1985 */ 1986 private void resetSplitParent(HbckRegionInfo hi) throws IOException { 1987 RowMutations mutations = new RowMutations(hi.getMetaEntry().getRegionName()); 1988 Delete d = new Delete(hi.getMetaEntry().getRegionName()); 1989 d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER); 1990 d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER); 1991 mutations.add(d); 1992 1993 RegionInfo hri = RegionInfoBuilder.newBuilder(hi.getMetaEntry()) 1994 .setOffline(false) 1995 .setSplit(false) 1996 .build(); 1997 Put p = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime()); 1998 mutations.add(p); 1999 2000 meta.mutateRow(mutations); 2001 LOG.info("Reset split parent " + hi.getMetaEntry().getRegionNameAsString() + " in META"); 2002 } 2003 2004 /** 2005 * This backwards-compatibility wrapper for permanently offlining a region 2006 * that should not be alive. If the region server does not support the 2007 * "offline" method, it will use the closest unassign method instead. This 2008 * will basically work until one attempts to disable or delete the affected 2009 * table. The problem has to do with in-memory only master state, so 2010 * restarting the HMaster or failing over to another should fix this. 2011 */ 2012 void offline(byte[] regionName) throws IOException { 2013 String regionString = Bytes.toStringBinary(regionName); 2014 if (!rsSupportsOffline) { 2015 LOG.warn( 2016 "Using unassign region " + regionString + " instead of using offline method, you should" + 2017 " restart HMaster after these repairs"); 2018 admin.unassign(regionName, true); 2019 return; 2020 } 2021 2022 // first time we assume the rs's supports #offline. 2023 try { 2024 LOG.info("Offlining region " + regionString); 2025 admin.offline(regionName); 2026 } catch (IOException ioe) { 2027 String notFoundMsg = "java.lang.NoSuchMethodException: " + 2028 "org.apache.hadoop.hbase.master.HMaster.offline([B)"; 2029 if (ioe.getMessage().contains(notFoundMsg)) { 2030 LOG.warn("Using unassign region " + regionString + 2031 " instead of using offline method, you should" + 2032 " restart HMaster after these repairs"); 2033 rsSupportsOffline = false; // in the future just use unassign 2034 admin.unassign(regionName, true); 2035 return; 2036 } 2037 throw ioe; 2038 } 2039 } 2040 2041 /** 2042 * Attempts to undeploy a region from a region server based in information in 2043 * META. Any operations that modify the file system should make sure that 2044 * its corresponding region is not deployed to prevent data races. 2045 * 2046 * A separate call is required to update the master in-memory region state 2047 * kept in the AssignementManager. Because disable uses this state instead of 2048 * that found in META, we can't seem to cleanly disable/delete tables that 2049 * have been hbck fixed. When used on a version of HBase that does not have 2050 * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master 2051 * restart or failover may be required. 2052 */ 2053 void closeRegion(HbckRegionInfo hi) throws IOException, InterruptedException { 2054 if (hi.getMetaEntry() == null && hi.getHdfsEntry() == null) { 2055 undeployRegions(hi); 2056 return; 2057 } 2058 2059 // get assignment info and hregioninfo from meta. 2060 Get get = new Get(hi.getRegionName()); 2061 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); 2062 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); 2063 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER); 2064 // also get the locations of the replicas to close if the primary region is being closed 2065 if (hi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { 2066 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication(); 2067 for (int i = 0; i < numReplicas; i++) { 2068 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i)); 2069 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i)); 2070 } 2071 } 2072 Result r = meta.get(get); 2073 RegionLocations rl = MetaTableAccessor.getRegionLocations(r); 2074 if (rl == null) { 2075 LOG.warn("Unable to close region " + hi.getRegionNameAsString() + 2076 " since meta does not have handle to reach it"); 2077 return; 2078 } 2079 for (HRegionLocation h : rl.getRegionLocations()) { 2080 ServerName serverName = h.getServerName(); 2081 if (serverName == null) { 2082 errors.reportError("Unable to close region " 2083 + hi.getRegionNameAsString() + " because meta does not " 2084 + "have handle to reach it."); 2085 continue; 2086 } 2087 RegionInfo hri = h.getRegionInfo(); 2088 if (hri == null) { 2089 LOG.warn("Unable to close region " + hi.getRegionNameAsString() 2090 + " because hbase:meta had invalid or missing " 2091 + HConstants.CATALOG_FAMILY_STR + ":" 2092 + Bytes.toString(HConstants.REGIONINFO_QUALIFIER) 2093 + " qualifier value."); 2094 continue; 2095 } 2096 // close the region -- close files and remove assignment 2097 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri); 2098 } 2099 } 2100 2101 private void undeployRegions(HbckRegionInfo hi) throws IOException, InterruptedException { 2102 undeployRegionsForHbi(hi); 2103 // undeploy replicas of the region (but only if the method is invoked for the primary) 2104 if (hi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 2105 return; 2106 } 2107 int numReplicas = admin.getDescriptor(hi.getTableName()).getRegionReplication(); 2108 for (int i = 1; i < numReplicas; i++) { 2109 if (hi.getPrimaryHRIForDeployedReplica() == null) continue; 2110 RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica( 2111 hi.getPrimaryHRIForDeployedReplica(), i); 2112 HbckRegionInfo h = regionInfoMap.get(hri.getEncodedName()); 2113 if (h != null) { 2114 undeployRegionsForHbi(h); 2115 //set skip checks; we undeployed it, and we don't want to evaluate this anymore 2116 //in consistency checks 2117 h.setSkipChecks(true); 2118 } 2119 } 2120 } 2121 2122 private void undeployRegionsForHbi(HbckRegionInfo hi) throws IOException, InterruptedException { 2123 for (HbckRegionInfo.OnlineEntry rse : hi.getOnlineEntries()) { 2124 LOG.debug("Undeploy region " + rse.getRegionInfo() + " from " + rse.getServerName()); 2125 try { 2126 HBaseFsckRepair 2127 .closeRegionSilentlyAndWait(connection, rse.getServerName(), rse.getRegionInfo()); 2128 offline(rse.getRegionInfo().getRegionName()); 2129 } catch (IOException ioe) { 2130 LOG.warn("Got exception when attempting to offline region " 2131 + Bytes.toString(rse.getRegionInfo().getRegionName()), ioe); 2132 } 2133 } 2134 } 2135 2136 private void tryAssignmentRepair(HbckRegionInfo hbi, String msg) throws IOException, 2137 KeeperException, InterruptedException { 2138 // If we are trying to fix the errors 2139 if (shouldFixAssignments()) { 2140 errors.print(msg); 2141 undeployRegions(hbi); 2142 setShouldRerun(); 2143 RegionInfo hri = hbi.getHdfsHRI(); 2144 if (hri == null) { 2145 hri = hbi.getMetaEntry(); 2146 } 2147 HBaseFsckRepair.fixUnassigned(admin, hri); 2148 HBaseFsckRepair.waitUntilAssigned(admin, hri); 2149 2150 // also assign replicas if needed (do it only when this call operates on a primary replica) 2151 if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) return; 2152 int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication(); 2153 for (int i = 1; i < replicationCount; i++) { 2154 hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i); 2155 HbckRegionInfo h = regionInfoMap.get(hri.getEncodedName()); 2156 if (h != null) { 2157 undeployRegions(h); 2158 //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore 2159 //in consistency checks 2160 h.setSkipChecks(true); 2161 } 2162 HBaseFsckRepair.fixUnassigned(admin, hri); 2163 HBaseFsckRepair.waitUntilAssigned(admin, hri); 2164 } 2165 2166 } 2167 } 2168 2169 /** 2170 * Check a single region for consistency and correct deployment. 2171 */ 2172 private void checkRegionConsistency(final String key, final HbckRegionInfo hbi) 2173 throws IOException, KeeperException, InterruptedException { 2174 2175 if (hbi.isSkipChecks()) return; 2176 String descriptiveName = hbi.toString(); 2177 boolean inMeta = hbi.getMetaEntry() != null; 2178 // In case not checking HDFS, assume the region is on HDFS 2179 boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null; 2180 boolean hasMetaAssignment = inMeta && hbi.getMetaEntry().regionServer != null; 2181 boolean isDeployed = !hbi.getDeployedOn().isEmpty(); 2182 boolean isMultiplyDeployed = hbi.getDeployedOn().size() > 1; 2183 boolean deploymentMatchesMeta = 2184 hasMetaAssignment && isDeployed && !isMultiplyDeployed && 2185 hbi.getMetaEntry().regionServer.equals(hbi.getDeployedOn().get(0)); 2186 boolean splitParent = 2187 inMeta && hbi.getMetaEntry().isSplit() && hbi.getMetaEntry().isOffline(); 2188 boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.getMetaEntry().getTable()); 2189 boolean recentlyModified = inHdfs && 2190 hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime(); 2191 2192 // ========== First the healthy cases ============= 2193 if (hbi.containsOnlyHdfsEdits()) { 2194 return; 2195 } 2196 if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) { 2197 return; 2198 } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) { 2199 LOG.info("Region " + descriptiveName + " is in META, and in a disabled " + 2200 "tabled that is not deployed"); 2201 return; 2202 } else if (recentlyModified) { 2203 LOG.warn("Region " + descriptiveName + " was recently modified -- skipping"); 2204 return; 2205 } 2206 // ========== Cases where the region is not in hbase:meta ============= 2207 else if (!inMeta && !inHdfs && !isDeployed) { 2208 // We shouldn't have record of this region at all then! 2209 assert false : "Entry for region with no data"; 2210 } else if (!inMeta && !inHdfs && isDeployed) { 2211 errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region " 2212 + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " + 2213 "deployed on " + Joiner.on(", ").join(hbi.getDeployedOn())); 2214 if (shouldFixAssignments()) { 2215 undeployRegions(hbi); 2216 } 2217 2218 } else if (!inMeta && inHdfs && !isDeployed) { 2219 if (hbi.isMerged()) { 2220 // This region has already been merged, the remaining hdfs file will be 2221 // cleaned by CatalogJanitor later 2222 hbi.setSkipChecks(true); 2223 LOG.info("Region " + descriptiveName 2224 + " got merge recently, its file(s) will be cleaned by CatalogJanitor later"); 2225 return; 2226 } 2227 errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region " 2228 + descriptiveName + " on HDFS, but not listed in hbase:meta " + 2229 "or deployed on any region server"); 2230 // restore region consistency of an adopted orphan 2231 if (shouldFixMeta()) { 2232 if (!hbi.isHdfsRegioninfoPresent()) { 2233 LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired" 2234 + " in table integrity repair phase if -fixHdfsOrphans was" + 2235 " used."); 2236 return; 2237 } 2238 2239 RegionInfo hri = hbi.getHdfsHRI(); 2240 HbckTableInfo tableInfo = tablesInfo.get(hri.getTable()); 2241 2242 for (RegionInfo region : tableInfo.getRegionsFromMeta(this.regionInfoMap)) { 2243 if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0 2244 && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(), 2245 hri.getEndKey()) >= 0) 2246 && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) { 2247 if(region.isSplit() || region.isOffline()) continue; 2248 Path regionDir = hbi.getHdfsRegionDir(); 2249 FileSystem fs = regionDir.getFileSystem(getConf()); 2250 List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir); 2251 for (Path familyDir : familyDirs) { 2252 List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir); 2253 for (Path referenceFilePath : referenceFilePaths) { 2254 Path parentRegionDir = 2255 StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent(); 2256 if (parentRegionDir.toString().endsWith(region.getEncodedName())) { 2257 LOG.warn(hri + " start and stop keys are in the range of " + region 2258 + ". The region might not be cleaned up from hdfs when region " + region 2259 + " split failed. Hence deleting from hdfs."); 2260 HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs, 2261 regionDir.getParent(), hri); 2262 return; 2263 } 2264 } 2265 } 2266 } 2267 } 2268 LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI()); 2269 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication(); 2270 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(), 2271 admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) 2272 .getLiveServerMetrics().keySet(), numReplicas); 2273 2274 tryAssignmentRepair(hbi, "Trying to reassign region..."); 2275 } 2276 2277 } else if (!inMeta && inHdfs && isDeployed) { 2278 errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName 2279 + " not in META, but deployed on " + Joiner.on(", ").join(hbi.getDeployedOn())); 2280 debugLsr(hbi.getHdfsRegionDir()); 2281 if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 2282 // for replicas, this means that we should undeploy the region (we would have 2283 // gone over the primaries and fixed meta holes in first phase under 2284 // checkAndFixConsistency; we shouldn't get the condition !inMeta at 2285 // this stage unless unwanted replica) 2286 if (shouldFixAssignments()) { 2287 undeployRegionsForHbi(hbi); 2288 } 2289 } 2290 if (shouldFixMeta() && hbi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { 2291 if (!hbi.isHdfsRegioninfoPresent()) { 2292 LOG.error("This should have been repaired in table integrity repair phase"); 2293 return; 2294 } 2295 2296 LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI()); 2297 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication(); 2298 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(), 2299 admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) 2300 .getLiveServerMetrics().keySet(), numReplicas); 2301 tryAssignmentRepair(hbi, "Trying to fix unassigned region..."); 2302 } 2303 2304 // ========== Cases where the region is in hbase:meta ============= 2305 } else if (inMeta && inHdfs && !isDeployed && splitParent) { 2306 // check whether this is an actual error, or just transient state where parent 2307 // is not cleaned 2308 if (hbi.getMetaEntry().splitA != null && hbi.getMetaEntry().splitB != null) { 2309 // check that split daughters are there 2310 HbckRegionInfo infoA = this.regionInfoMap.get(hbi.getMetaEntry().splitA.getEncodedName()); 2311 HbckRegionInfo infoB = this.regionInfoMap.get(hbi.getMetaEntry().splitB.getEncodedName()); 2312 if (infoA != null && infoB != null) { 2313 // we already processed or will process daughters. Move on, nothing to see here. 2314 hbi.setSkipChecks(true); 2315 return; 2316 } 2317 } 2318 2319 // For Replica region, we need to do a similar check. If replica is not split successfully, 2320 // error is going to be reported against primary daughter region. 2321 if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 2322 LOG.info("Region " + descriptiveName + " is a split parent in META, in HDFS, " 2323 + "and not deployed on any region server. This may be transient."); 2324 hbi.setSkipChecks(true); 2325 return; 2326 } 2327 2328 errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region " 2329 + descriptiveName + " is a split parent in META, in HDFS, " 2330 + "and not deployed on any region server. This could be transient, " 2331 + "consider to run the catalog janitor first!"); 2332 if (shouldFixSplitParents()) { 2333 setShouldRerun(); 2334 resetSplitParent(hbi); 2335 } 2336 } else if (inMeta && !inHdfs && !isDeployed) { 2337 errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region " 2338 + descriptiveName + " found in META, but not in HDFS " 2339 + "or deployed on any region server."); 2340 if (shouldFixMeta()) { 2341 deleteMetaRegion(hbi); 2342 } 2343 } else if (inMeta && !inHdfs && isDeployed) { 2344 errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName 2345 + " found in META, but not in HDFS, " + 2346 "and deployed on " + Joiner.on(", ").join(hbi.getDeployedOn())); 2347 // We treat HDFS as ground truth. Any information in meta is transient 2348 // and equivalent data can be regenerated. So, lets unassign and remove 2349 // these problems from META. 2350 if (shouldFixAssignments()) { 2351 errors.print("Trying to fix unassigned region..."); 2352 undeployRegions(hbi); 2353 } 2354 if (shouldFixMeta()) { 2355 // wait for it to complete 2356 deleteMetaRegion(hbi); 2357 } 2358 } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) { 2359 errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName 2360 + " not deployed on any region server."); 2361 tryAssignmentRepair(hbi, "Trying to fix unassigned region..."); 2362 } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) { 2363 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED, 2364 "Region " + descriptiveName + " should not be deployed according " + 2365 "to META, but is deployed on " + Joiner.on(", ").join(hbi.getDeployedOn())); 2366 if (shouldFixAssignments()) { 2367 errors.print("Trying to close the region " + descriptiveName); 2368 setShouldRerun(); 2369 HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry(), hbi.getDeployedOn()); 2370 } 2371 } else if (inMeta && inHdfs && isMultiplyDeployed) { 2372 errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName 2373 + " is listed in hbase:meta on region server " + hbi.getMetaEntry().regionServer 2374 + " but is multiply assigned to region servers " + 2375 Joiner.on(", ").join(hbi.getDeployedOn())); 2376 // If we are trying to fix the errors 2377 if (shouldFixAssignments()) { 2378 errors.print("Trying to fix assignment error..."); 2379 setShouldRerun(); 2380 HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry(), hbi.getDeployedOn()); 2381 } 2382 } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) { 2383 errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region " 2384 + descriptiveName + " listed in hbase:meta on region server " + 2385 hbi.getMetaEntry().regionServer + " but found on region server " + 2386 hbi.getDeployedOn().get(0)); 2387 // If we are trying to fix the errors 2388 if (shouldFixAssignments()) { 2389 errors.print("Trying to fix assignment error..."); 2390 setShouldRerun(); 2391 HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry(), hbi.getDeployedOn()); 2392 HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI()); 2393 } 2394 } else { 2395 errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName + 2396 " is in an unforeseen state:" + 2397 " inMeta=" + inMeta + 2398 " inHdfs=" + inHdfs + 2399 " isDeployed=" + isDeployed + 2400 " isMultiplyDeployed=" + isMultiplyDeployed + 2401 " deploymentMatchesMeta=" + deploymentMatchesMeta + 2402 " shouldBeDeployed=" + shouldBeDeployed); 2403 } 2404 } 2405 2406 /** 2407 * Checks tables integrity. Goes over all regions and scans the tables. 2408 * Collects all the pieces for each table and checks if there are missing, 2409 * repeated or overlapping ones. 2410 * @throws IOException 2411 */ 2412 SortedMap<TableName, HbckTableInfo> checkIntegrity() throws IOException { 2413 tablesInfo = new TreeMap<>(); 2414 LOG.debug("There are " + regionInfoMap.size() + " region info entries"); 2415 for (HbckRegionInfo hbi : regionInfoMap.values()) { 2416 // Check only valid, working regions 2417 if (hbi.getMetaEntry() == null) { 2418 // this assumes that consistency check has run loadMetaEntry 2419 Path p = hbi.getHdfsRegionDir(); 2420 if (p == null) { 2421 errors.report("No regioninfo in Meta or HDFS. " + hbi); 2422 } 2423 2424 // TODO test. 2425 continue; 2426 } 2427 if (hbi.getMetaEntry().regionServer == null) { 2428 errors.detail("Skipping region because no region server: " + hbi); 2429 continue; 2430 } 2431 if (hbi.getMetaEntry().isOffline()) { 2432 errors.detail("Skipping region because it is offline: " + hbi); 2433 continue; 2434 } 2435 if (hbi.containsOnlyHdfsEdits()) { 2436 errors.detail("Skipping region because it only contains edits" + hbi); 2437 continue; 2438 } 2439 2440 // Missing regionDir or over-deployment is checked elsewhere. Include 2441 // these cases in modTInfo, so we can evaluate those regions as part of 2442 // the region chain in META 2443 //if (hbi.foundRegionDir == null) continue; 2444 //if (hbi.deployedOn.size() != 1) continue; 2445 if (hbi.getDeployedOn().isEmpty()) { 2446 continue; 2447 } 2448 2449 // We should be safe here 2450 TableName tableName = hbi.getMetaEntry().getTable(); 2451 HbckTableInfo modTInfo = tablesInfo.get(tableName); 2452 if (modTInfo == null) { 2453 modTInfo = new HbckTableInfo(tableName, this); 2454 } 2455 for (ServerName server : hbi.getDeployedOn()) { 2456 modTInfo.addServer(server); 2457 } 2458 2459 if (!hbi.isSkipChecks()) { 2460 modTInfo.addRegionInfo(hbi); 2461 } 2462 2463 tablesInfo.put(tableName, modTInfo); 2464 } 2465 2466 loadTableInfosForTablesWithNoRegion(); 2467 2468 logParallelMerge(); 2469 for (HbckTableInfo tInfo : tablesInfo.values()) { 2470 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors); 2471 if (!tInfo.checkRegionChain(handler)) { 2472 errors.report("Found inconsistency in table " + tInfo.getName()); 2473 } 2474 } 2475 return tablesInfo; 2476 } 2477 2478 /** Loads table info's for tables that may not have been included, since there are no 2479 * regions reported for the table, but table dir is there in hdfs 2480 */ 2481 private void loadTableInfosForTablesWithNoRegion() throws IOException { 2482 Map<String, TableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll(); 2483 for (TableDescriptor htd : allTables.values()) { 2484 if (checkMetaOnly && !htd.isMetaTable()) { 2485 continue; 2486 } 2487 2488 TableName tableName = htd.getTableName(); 2489 if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) { 2490 HbckTableInfo tableInfo = new HbckTableInfo(tableName, this); 2491 tableInfo.htds.add(htd); 2492 tablesInfo.put(htd.getTableName(), tableInfo); 2493 } 2494 } 2495 } 2496 2497 /** 2498 * Merge hdfs data by moving from contained HbckRegionInfo into targetRegionDir. 2499 * @return number of file move fixes done to merge regions. 2500 */ 2501 public int mergeRegionDirs(Path targetRegionDir, HbckRegionInfo contained) throws IOException { 2502 int fileMoves = 0; 2503 String thread = Thread.currentThread().getName(); 2504 LOG.debug("[" + thread + "] Contained region dir after close and pause"); 2505 debugLsr(contained.getHdfsRegionDir()); 2506 2507 // rename the contained into the container. 2508 FileSystem fs = targetRegionDir.getFileSystem(getConf()); 2509 FileStatus[] dirs = null; 2510 try { 2511 dirs = fs.listStatus(contained.getHdfsRegionDir()); 2512 } catch (FileNotFoundException fnfe) { 2513 // region we are attempting to merge in is not present! Since this is a merge, there is 2514 // no harm skipping this region if it does not exist. 2515 if (!fs.exists(contained.getHdfsRegionDir())) { 2516 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir() 2517 + " is missing. Assuming already sidelined or moved."); 2518 } else { 2519 sidelineRegionDir(fs, contained); 2520 } 2521 return fileMoves; 2522 } 2523 2524 if (dirs == null) { 2525 if (!fs.exists(contained.getHdfsRegionDir())) { 2526 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir() 2527 + " already sidelined."); 2528 } else { 2529 sidelineRegionDir(fs, contained); 2530 } 2531 return fileMoves; 2532 } 2533 2534 for (FileStatus cf : dirs) { 2535 Path src = cf.getPath(); 2536 Path dst = new Path(targetRegionDir, src.getName()); 2537 2538 if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) { 2539 // do not copy the old .regioninfo file. 2540 continue; 2541 } 2542 2543 if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) { 2544 // do not copy the .oldlogs files 2545 continue; 2546 } 2547 2548 LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst); 2549 // FileSystem.rename is inconsistent with directories -- if the 2550 // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir, 2551 // it moves the src into the dst dir resulting in (foo/a/b). If 2552 // the dst does not exist, and the src a dir, src becomes dst. (foo/b) 2553 for (FileStatus hfile : fs.listStatus(src)) { 2554 boolean success = fs.rename(hfile.getPath(), dst); 2555 if (success) { 2556 fileMoves++; 2557 } 2558 } 2559 LOG.debug("[" + thread + "] Sideline directory contents:"); 2560 debugLsr(targetRegionDir); 2561 } 2562 2563 // if all success. 2564 sidelineRegionDir(fs, contained); 2565 LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " + 2566 getSidelineDir()); 2567 debugLsr(contained.getHdfsRegionDir()); 2568 2569 return fileMoves; 2570 } 2571 2572 2573 static class WorkItemOverlapMerge implements Callable<Void> { 2574 private TableIntegrityErrorHandler handler; 2575 Collection<HbckRegionInfo> overlapgroup; 2576 2577 WorkItemOverlapMerge(Collection<HbckRegionInfo> overlapgroup, 2578 TableIntegrityErrorHandler handler) { 2579 this.handler = handler; 2580 this.overlapgroup = overlapgroup; 2581 } 2582 2583 @Override 2584 public Void call() throws Exception { 2585 handler.handleOverlapGroup(overlapgroup); 2586 return null; 2587 } 2588 }; 2589 2590 /** 2591 * Return a list of user-space table names whose metadata have not been 2592 * modified in the last few milliseconds specified by timelag 2593 * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER, 2594 * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last 2595 * milliseconds specified by timelag, then the table is a candidate to be returned. 2596 * @return tables that have not been modified recently 2597 * @throws IOException if an error is encountered 2598 */ 2599 TableDescriptor[] getTables(AtomicInteger numSkipped) { 2600 List<TableName> tableNames = new ArrayList<>(); 2601 long now = EnvironmentEdgeManager.currentTime(); 2602 2603 for (HbckRegionInfo hbi : regionInfoMap.values()) { 2604 HbckRegionInfo.MetaEntry info = hbi.getMetaEntry(); 2605 2606 // if the start key is zero, then we have found the first region of a table. 2607 // pick only those tables that were not modified in the last few milliseconds. 2608 if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) { 2609 if (info.modTime + timelag < now) { 2610 tableNames.add(info.getTable()); 2611 } else { 2612 numSkipped.incrementAndGet(); // one more in-flux table 2613 } 2614 } 2615 } 2616 return getTableDescriptors(tableNames); 2617 } 2618 2619 TableDescriptor[] getTableDescriptors(List<TableName> tableNames) { 2620 LOG.info("getTableDescriptors == tableNames => " + tableNames); 2621 try (Connection conn = ConnectionFactory.createConnection(getConf()); 2622 Admin admin = conn.getAdmin()) { 2623 List<TableDescriptor> tds = admin.listTableDescriptors(tableNames); 2624 return tds.toArray(new TableDescriptor[tds.size()]); 2625 } catch (IOException e) { 2626 LOG.debug("Exception getting table descriptors", e); 2627 } 2628 return new TableDescriptor[0]; 2629 } 2630 2631 /** 2632 * Gets the entry in regionInfo corresponding to the the given encoded 2633 * region name. If the region has not been seen yet, a new entry is added 2634 * and returned. 2635 */ 2636 private synchronized HbckRegionInfo getOrCreateInfo(String name) { 2637 HbckRegionInfo hbi = regionInfoMap.get(name); 2638 if (hbi == null) { 2639 hbi = new HbckRegionInfo(null); 2640 regionInfoMap.put(name, hbi); 2641 } 2642 return hbi; 2643 } 2644 2645 private void checkAndFixReplication() throws ReplicationException { 2646 ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, errors); 2647 checker.checkUnDeletedQueues(); 2648 2649 if (checker.hasUnDeletedQueues() && this.fixReplication) { 2650 checker.fixUnDeletedQueues(); 2651 setShouldRerun(); 2652 } 2653 } 2654 2655 /** 2656 * Check values in regionInfo for hbase:meta 2657 * Check if zero or more than one regions with hbase:meta are found. 2658 * If there are inconsistencies (i.e. zero or more than one regions 2659 * pretend to be holding the hbase:meta) try to fix that and report an error. 2660 * @throws IOException from HBaseFsckRepair functions 2661 * @throws KeeperException 2662 * @throws InterruptedException 2663 */ 2664 boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException { 2665 Map<Integer, HbckRegionInfo> metaRegions = new HashMap<>(); 2666 for (HbckRegionInfo value : regionInfoMap.values()) { 2667 if (value.getMetaEntry() != null && value.getMetaEntry().isMetaRegion()) { 2668 metaRegions.put(value.getReplicaId(), value); 2669 } 2670 } 2671 int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME) 2672 .getRegionReplication(); 2673 boolean noProblem = true; 2674 // There will be always entries in regionInfoMap corresponding to hbase:meta & its replicas 2675 // Check the deployed servers. It should be exactly one server for each replica. 2676 for (int i = 0; i < metaReplication; i++) { 2677 HbckRegionInfo metaHbckRegionInfo = metaRegions.remove(i); 2678 List<ServerName> servers = new ArrayList<>(); 2679 if (metaHbckRegionInfo != null) { 2680 servers = metaHbckRegionInfo.getDeployedOn(); 2681 } 2682 if (servers.size() != 1) { 2683 noProblem = false; 2684 if (servers.isEmpty()) { 2685 assignMetaReplica(i); 2686 } else if (servers.size() > 1) { 2687 errors 2688 .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " + 2689 metaHbckRegionInfo.getReplicaId() + " is found on more than one region."); 2690 if (shouldFixAssignments()) { 2691 errors.print("Trying to fix a problem with hbase:meta, replicaId " + 2692 metaHbckRegionInfo.getReplicaId() + ".."); 2693 setShouldRerun(); 2694 // try fix it (treat is a dupe assignment) 2695 HBaseFsckRepair 2696 .fixMultiAssignment(connection, metaHbckRegionInfo.getMetaEntry(), servers); 2697 } 2698 } 2699 } 2700 } 2701 // unassign whatever is remaining in metaRegions. They are excess replicas. 2702 for (Map.Entry<Integer, HbckRegionInfo> entry : metaRegions.entrySet()) { 2703 noProblem = false; 2704 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED, 2705 "hbase:meta replicas are deployed in excess. Configured " + metaReplication + 2706 ", deployed " + metaRegions.size()); 2707 if (shouldFixAssignments()) { 2708 errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() + 2709 " of hbase:meta.."); 2710 setShouldRerun(); 2711 unassignMetaReplica(entry.getValue()); 2712 } 2713 } 2714 // if noProblem is false, rerun hbck with hopefully fixed META 2715 // if noProblem is true, no errors, so continue normally 2716 return noProblem; 2717 } 2718 2719 private void unassignMetaReplica(HbckRegionInfo hi) 2720 throws IOException, InterruptedException, KeeperException { 2721 undeployRegions(hi); 2722 ZKUtil 2723 .deleteNode(zkw, zkw.getZNodePaths().getZNodeForReplica(hi.getMetaEntry().getReplicaId())); 2724 } 2725 2726 private void assignMetaReplica(int replicaId) 2727 throws IOException, KeeperException, InterruptedException { 2728 errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " + 2729 replicaId +" is not found on any region."); 2730 if (shouldFixAssignments()) { 2731 errors.print("Trying to fix a problem with hbase:meta.."); 2732 setShouldRerun(); 2733 // try to fix it (treat it as unassigned region) 2734 RegionInfo h = RegionReplicaUtil.getRegionInfoForReplica( 2735 RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId); 2736 HBaseFsckRepair.fixUnassigned(admin, h); 2737 HBaseFsckRepair.waitUntilAssigned(admin, h); 2738 } 2739 } 2740 2741 /** 2742 * Scan hbase:meta, adding all regions found to the regionInfo map. 2743 * @throws IOException if an error is encountered 2744 */ 2745 boolean loadMetaEntries() throws IOException { 2746 MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() { 2747 int countRecord = 1; 2748 2749 // comparator to sort KeyValues with latest modtime 2750 final Comparator<Cell> comp = new Comparator<Cell>() { 2751 @Override 2752 public int compare(Cell k1, Cell k2) { 2753 return Long.compare(k1.getTimestamp(), k2.getTimestamp()); 2754 } 2755 }; 2756 2757 @Override 2758 public boolean visit(Result result) throws IOException { 2759 try { 2760 2761 // record the latest modification of this META record 2762 long ts = Collections.max(result.listCells(), comp).getTimestamp(); 2763 RegionLocations rl = MetaTableAccessor.getRegionLocations(result); 2764 if (rl == null) { 2765 emptyRegionInfoQualifiers.add(result); 2766 errors.reportError(ERROR_CODE.EMPTY_META_CELL, 2767 "Empty REGIONINFO_QUALIFIER found in hbase:meta"); 2768 return true; 2769 } 2770 ServerName sn = null; 2771 if (rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID) == null || 2772 rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) { 2773 emptyRegionInfoQualifiers.add(result); 2774 errors.reportError(ERROR_CODE.EMPTY_META_CELL, 2775 "Empty REGIONINFO_QUALIFIER found in hbase:meta"); 2776 return true; 2777 } 2778 RegionInfo hri = rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegionInfo(); 2779 if (!(isTableIncluded(hri.getTable()) 2780 || hri.isMetaRegion())) { 2781 return true; 2782 } 2783 PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(result); 2784 for (HRegionLocation h : rl.getRegionLocations()) { 2785 if (h == null || h.getRegionInfo() == null) { 2786 continue; 2787 } 2788 sn = h.getServerName(); 2789 hri = h.getRegionInfo(); 2790 2791 HbckRegionInfo.MetaEntry m = null; 2792 if (hri.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { 2793 m = new HbckRegionInfo.MetaEntry(hri, sn, ts, daughters.getFirst(), 2794 daughters.getSecond()); 2795 } else { 2796 m = new HbckRegionInfo.MetaEntry(hri, sn, ts, null, null); 2797 } 2798 HbckRegionInfo previous = regionInfoMap.get(hri.getEncodedName()); 2799 if (previous == null) { 2800 regionInfoMap.put(hri.getEncodedName(), new HbckRegionInfo(m)); 2801 } else if (previous.getMetaEntry() == null) { 2802 previous.setMetaEntry(m); 2803 } else { 2804 throw new IOException("Two entries in hbase:meta are same " + previous); 2805 } 2806 } 2807 List<RegionInfo> mergeParents = MetaTableAccessor.getMergeRegions(result.rawCells()); 2808 if (mergeParents != null) { 2809 for (RegionInfo mergeRegion : mergeParents) { 2810 if (mergeRegion != null) { 2811 // This region is already being merged 2812 HbckRegionInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName()); 2813 hbInfo.setMerged(true); 2814 } 2815 } 2816 } 2817 2818 // show proof of progress to the user, once for every 100 records. 2819 if (countRecord % 100 == 0) { 2820 errors.progress(); 2821 } 2822 countRecord++; 2823 return true; 2824 } catch (RuntimeException e) { 2825 LOG.error("Result=" + result); 2826 throw e; 2827 } 2828 } 2829 }; 2830 if (!checkMetaOnly) { 2831 // Scan hbase:meta to pick up user regions 2832 MetaTableAccessor.fullScanRegions(connection, visitor); 2833 } 2834 2835 errors.print(""); 2836 return true; 2837 } 2838 2839 /** 2840 * Prints summary of all tables found on the system. 2841 */ 2842 private void printTableSummary(SortedMap<TableName, HbckTableInfo> tablesInfo) { 2843 StringBuilder sb = new StringBuilder(); 2844 int numOfSkippedRegions; 2845 errors.print("Summary:"); 2846 for (HbckTableInfo tInfo : tablesInfo.values()) { 2847 numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ? 2848 skippedRegions.get(tInfo.getName()).size() : 0; 2849 2850 if (errors.tableHasErrors(tInfo)) { 2851 errors.print("Table " + tInfo.getName() + " is inconsistent."); 2852 } else if (numOfSkippedRegions > 0){ 2853 errors.print("Table " + tInfo.getName() + " is okay (with " 2854 + numOfSkippedRegions + " skipped regions)."); 2855 } 2856 else { 2857 errors.print("Table " + tInfo.getName() + " is okay."); 2858 } 2859 errors.print(" Number of regions: " + tInfo.getNumRegions()); 2860 if (numOfSkippedRegions > 0) { 2861 Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName()); 2862 System.out.println(" Number of skipped regions: " + numOfSkippedRegions); 2863 System.out.println(" List of skipped regions:"); 2864 for(String sr : skippedRegionStrings) { 2865 System.out.println(" " + sr); 2866 } 2867 } 2868 sb.setLength(0); // clear out existing buffer, if any. 2869 sb.append(" Deployed on: "); 2870 for (ServerName server : tInfo.deployedOn) { 2871 sb.append(" " + server.toString()); 2872 } 2873 errors.print(sb.toString()); 2874 } 2875 } 2876 2877 static HbckErrorReporter getErrorReporter(final Configuration conf) 2878 throws ClassNotFoundException { 2879 Class<? extends HbckErrorReporter> reporter = 2880 conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, 2881 HbckErrorReporter.class); 2882 return ReflectionUtils.newInstance(reporter, conf); 2883 } 2884 2885 static class PrintingErrorReporter implements HbckErrorReporter { 2886 public int errorCount = 0; 2887 private int showProgress; 2888 // How frequently calls to progress() will create output 2889 private static final int progressThreshold = 100; 2890 2891 Set<HbckTableInfo> errorTables = new HashSet<>(); 2892 2893 // for use by unit tests to verify which errors were discovered 2894 private ArrayList<ERROR_CODE> errorList = new ArrayList<>(); 2895 2896 @Override 2897 public void clear() { 2898 errorTables.clear(); 2899 errorList.clear(); 2900 errorCount = 0; 2901 } 2902 2903 @Override 2904 public synchronized void reportError(ERROR_CODE errorCode, String message) { 2905 if (errorCode == ERROR_CODE.WRONG_USAGE) { 2906 System.err.println(message); 2907 return; 2908 } 2909 2910 errorList.add(errorCode); 2911 if (!summary) { 2912 System.out.println("ERROR: " + message); 2913 } 2914 errorCount++; 2915 showProgress = 0; 2916 } 2917 2918 @Override 2919 public synchronized void reportError(ERROR_CODE errorCode, String message, 2920 HbckTableInfo table) { 2921 errorTables.add(table); 2922 reportError(errorCode, message); 2923 } 2924 2925 @Override 2926 public synchronized void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table, 2927 HbckRegionInfo info) { 2928 errorTables.add(table); 2929 String reference = "(region " + info.getRegionNameAsString() + ")"; 2930 reportError(errorCode, reference + " " + message); 2931 } 2932 2933 @Override 2934 public synchronized void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table, 2935 HbckRegionInfo info1, HbckRegionInfo info2) { 2936 errorTables.add(table); 2937 String reference = "(regions " + info1.getRegionNameAsString() 2938 + " and " + info2.getRegionNameAsString() + ")"; 2939 reportError(errorCode, reference + " " + message); 2940 } 2941 2942 @Override 2943 public synchronized void reportError(String message) { 2944 reportError(ERROR_CODE.UNKNOWN, message); 2945 } 2946 2947 /** 2948 * Report error information, but do not increment the error count. Intended for cases 2949 * where the actual error would have been reported previously. 2950 * @param message 2951 */ 2952 @Override 2953 public synchronized void report(String message) { 2954 if (! summary) { 2955 System.out.println("ERROR: " + message); 2956 } 2957 showProgress = 0; 2958 } 2959 2960 @Override 2961 public synchronized int summarize() { 2962 System.out.println(Integer.toString(errorCount) + 2963 " inconsistencies detected."); 2964 if (errorCount == 0) { 2965 System.out.println("Status: OK"); 2966 return 0; 2967 } else { 2968 System.out.println("Status: INCONSISTENT"); 2969 return -1; 2970 } 2971 } 2972 2973 @Override 2974 public ArrayList<ERROR_CODE> getErrorList() { 2975 return errorList; 2976 } 2977 2978 @Override 2979 public synchronized void print(String message) { 2980 if (!summary) { 2981 System.out.println(message); 2982 } 2983 } 2984 2985 @Override 2986 public boolean tableHasErrors(HbckTableInfo table) { 2987 return errorTables.contains(table); 2988 } 2989 2990 @Override 2991 public void resetErrors() { 2992 errorCount = 0; 2993 } 2994 2995 @Override 2996 public synchronized void detail(String message) { 2997 if (details) { 2998 System.out.println(message); 2999 } 3000 showProgress = 0; 3001 } 3002 3003 @Override 3004 public synchronized void progress() { 3005 if (showProgress++ == progressThreshold) { 3006 if (!summary) { 3007 System.out.print("."); 3008 } 3009 showProgress = 0; 3010 } 3011 } 3012 } 3013 3014 /** 3015 * Contact a region server and get all information from it 3016 */ 3017 static class WorkItemRegion implements Callable<Void> { 3018 private final HBaseFsck hbck; 3019 private final ServerName rsinfo; 3020 private final HbckErrorReporter errors; 3021 private final ClusterConnection connection; 3022 3023 WorkItemRegion(HBaseFsck hbck, ServerName info, HbckErrorReporter errors, 3024 ClusterConnection connection) { 3025 this.hbck = hbck; 3026 this.rsinfo = info; 3027 this.errors = errors; 3028 this.connection = connection; 3029 } 3030 3031 @Override 3032 public synchronized Void call() throws IOException { 3033 errors.progress(); 3034 try { 3035 BlockingInterface server = connection.getAdmin(rsinfo); 3036 3037 // list all online regions from this region server 3038 List<RegionInfo> regions = ProtobufUtil.getOnlineRegions(server); 3039 regions = filterRegions(regions); 3040 3041 if (details) { 3042 errors.detail("RegionServer: " + rsinfo.getServerName() + 3043 " number of regions: " + regions.size()); 3044 for (RegionInfo rinfo: regions) { 3045 errors.detail(" " + rinfo.getRegionNameAsString() + 3046 " id: " + rinfo.getRegionId() + 3047 " encoded_name: " + rinfo.getEncodedName() + 3048 " start: " + Bytes.toStringBinary(rinfo.getStartKey()) + 3049 " end: " + Bytes.toStringBinary(rinfo.getEndKey())); 3050 } 3051 } 3052 3053 // check to see if the existence of this region matches the region in META 3054 3055 for (RegionInfo r : regions) { 3056 HbckRegionInfo hbi = hbck.getOrCreateInfo(r.getEncodedName()); 3057 hbi.addServer(r, rsinfo); 3058 } 3059 } catch (IOException e) { // unable to connect to the region server. 3060 errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() + 3061 " Unable to fetch region information. " + e); 3062 throw e; 3063 } 3064 return null; 3065 } 3066 3067 private List<RegionInfo> filterRegions(List<RegionInfo> regions) { 3068 List<RegionInfo> ret = Lists.newArrayList(); 3069 for (RegionInfo hri : regions) { 3070 if (hri.isMetaRegion() || (!hbck.checkMetaOnly 3071 && hbck.isTableIncluded(hri.getTable()))) { 3072 ret.add(hri); 3073 } 3074 } 3075 return ret; 3076 } 3077 } 3078 3079 /** 3080 * Contact hdfs and get all information about specified table directory into 3081 * regioninfo list. 3082 */ 3083 class WorkItemHdfsDir implements Callable<Void> { 3084 private FileStatus tableDir; 3085 private HbckErrorReporter errors; 3086 private FileSystem fs; 3087 3088 WorkItemHdfsDir(FileSystem fs, HbckErrorReporter errors, FileStatus status) { 3089 this.fs = fs; 3090 this.tableDir = status; 3091 this.errors = errors; 3092 } 3093 3094 @Override 3095 public synchronized Void call() throws InterruptedException, ExecutionException { 3096 final Vector<Exception> exceptions = new Vector<>(); 3097 3098 try { 3099 final FileStatus[] regionDirs = fs.listStatus(tableDir.getPath()); 3100 final List<Future<?>> futures = new ArrayList<>(regionDirs.length); 3101 3102 for (final FileStatus regionDir : regionDirs) { 3103 errors.progress(); 3104 final String encodedName = regionDir.getPath().getName(); 3105 // ignore directories that aren't hexadecimal 3106 if (!encodedName.toLowerCase(Locale.ROOT).matches("[0-9a-f]+")) { 3107 continue; 3108 } 3109 3110 if (!exceptions.isEmpty()) { 3111 break; 3112 } 3113 3114 futures.add(executor.submit(new Runnable() { 3115 @Override 3116 public void run() { 3117 try { 3118 LOG.debug("Loading region info from hdfs:"+ regionDir.getPath()); 3119 3120 Path regioninfoFile = new Path(regionDir.getPath(), HRegionFileSystem.REGION_INFO_FILE); 3121 boolean regioninfoFileExists = fs.exists(regioninfoFile); 3122 3123 if (!regioninfoFileExists) { 3124 // As tables become larger it is more and more likely that by the time you 3125 // reach a given region that it will be gone due to region splits/merges. 3126 if (!fs.exists(regionDir.getPath())) { 3127 LOG.warn("By the time we tried to process this region dir it was already gone: " 3128 + regionDir.getPath()); 3129 return; 3130 } 3131 } 3132 3133 HbckRegionInfo hbi = HBaseFsck.this.getOrCreateInfo(encodedName); 3134 HbckRegionInfo.HdfsEntry he = new HbckRegionInfo.HdfsEntry(); 3135 synchronized (hbi) { 3136 if (hbi.getHdfsRegionDir() != null) { 3137 errors.print("Directory " + encodedName + " duplicate??" + 3138 hbi.getHdfsRegionDir()); 3139 } 3140 3141 he.regionDir = regionDir.getPath(); 3142 he.regionDirModTime = regionDir.getModificationTime(); 3143 he.hdfsRegioninfoFilePresent = regioninfoFileExists; 3144 // we add to orphan list when we attempt to read .regioninfo 3145 3146 // Set a flag if this region contains only edits 3147 // This is special case if a region is left after split 3148 he.hdfsOnlyEdits = true; 3149 FileStatus[] subDirs = fs.listStatus(regionDir.getPath()); 3150 Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath()); 3151 for (FileStatus subDir : subDirs) { 3152 errors.progress(); 3153 String sdName = subDir.getPath().getName(); 3154 if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) { 3155 he.hdfsOnlyEdits = false; 3156 break; 3157 } 3158 } 3159 hbi.setHdfsEntry(he); 3160 } 3161 } catch (Exception e) { 3162 LOG.error("Could not load region dir", e); 3163 exceptions.add(e); 3164 } 3165 } 3166 })); 3167 } 3168 3169 // Ensure all pending tasks are complete (or that we run into an exception) 3170 for (Future<?> f : futures) { 3171 if (!exceptions.isEmpty()) { 3172 break; 3173 } 3174 try { 3175 f.get(); 3176 } catch (ExecutionException e) { 3177 LOG.error("Unexpected exec exception! Should've been caught already. (Bug?)", e); 3178 // Shouldn't happen, we already logged/caught any exceptions in the Runnable 3179 }; 3180 } 3181 } catch (IOException e) { 3182 LOG.error("Cannot execute WorkItemHdfsDir for " + tableDir, e); 3183 exceptions.add(e); 3184 } finally { 3185 if (!exceptions.isEmpty()) { 3186 errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: " 3187 + tableDir.getPath().getName() 3188 + " Unable to fetch all HDFS region information. "); 3189 // Just throw the first exception as an indication something bad happened 3190 // Don't need to propagate all the exceptions, we already logged them all anyway 3191 throw new ExecutionException("First exception in WorkItemHdfsDir", exceptions.firstElement()); 3192 } 3193 } 3194 return null; 3195 } 3196 } 3197 3198 /** 3199 * Contact hdfs and get all information about specified table directory into 3200 * regioninfo list. 3201 */ 3202 static class WorkItemHdfsRegionInfo implements Callable<Void> { 3203 private HbckRegionInfo hbi; 3204 private HBaseFsck hbck; 3205 private HbckErrorReporter errors; 3206 3207 WorkItemHdfsRegionInfo(HbckRegionInfo hbi, HBaseFsck hbck, HbckErrorReporter errors) { 3208 this.hbi = hbi; 3209 this.hbck = hbck; 3210 this.errors = errors; 3211 } 3212 3213 @Override 3214 public synchronized Void call() throws IOException { 3215 // only load entries that haven't been loaded yet. 3216 if (hbi.getHdfsHRI() == null) { 3217 try { 3218 errors.progress(); 3219 hbi.loadHdfsRegioninfo(hbck.getConf()); 3220 } catch (IOException ioe) { 3221 String msg = "Orphan region in HDFS: Unable to load .regioninfo from table " 3222 + hbi.getTableName() + " in hdfs dir " 3223 + hbi.getHdfsRegionDir() 3224 + "! It may be an invalid format or version file. Treating as " 3225 + "an orphaned regiondir."; 3226 errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg); 3227 try { 3228 hbck.debugLsr(hbi.getHdfsRegionDir()); 3229 } catch (IOException ioe2) { 3230 LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2); 3231 throw ioe2; 3232 } 3233 hbck.orphanHdfsDirs.add(hbi); 3234 throw ioe; 3235 } 3236 } 3237 return null; 3238 } 3239 }; 3240 3241 /** 3242 * Display the full report from fsck. This displays all live and dead region 3243 * servers, and all known regions. 3244 */ 3245 public static void setDisplayFullReport() { 3246 details = true; 3247 } 3248 3249 public static boolean shouldDisplayFullReport() { 3250 return details; 3251 } 3252 3253 /** 3254 * Set exclusive mode. 3255 */ 3256 public static void setForceExclusive() { 3257 forceExclusive = true; 3258 } 3259 3260 /** 3261 * Only one instance of hbck can modify HBase at a time. 3262 */ 3263 public boolean isExclusive() { 3264 return fixAny || forceExclusive; 3265 } 3266 3267 /** 3268 * Set summary mode. 3269 * Print only summary of the tables and status (OK or INCONSISTENT) 3270 */ 3271 static void setSummary() { 3272 summary = true; 3273 } 3274 3275 /** 3276 * Set hbase:meta check mode. 3277 * Print only info about hbase:meta table deployment/state 3278 */ 3279 void setCheckMetaOnly() { 3280 checkMetaOnly = true; 3281 } 3282 3283 /** 3284 * Set region boundaries check mode. 3285 */ 3286 void setRegionBoundariesCheck() { 3287 checkRegionBoundaries = true; 3288 } 3289 3290 /** 3291 * Set replication fix mode. 3292 */ 3293 public void setFixReplication(boolean shouldFix) { 3294 fixReplication = shouldFix; 3295 fixAny |= shouldFix; 3296 } 3297 3298 public void setCleanReplicationBarrier(boolean shouldClean) { 3299 cleanReplicationBarrier = shouldClean; 3300 } 3301 3302 /** 3303 * Check if we should rerun fsck again. This checks if we've tried to 3304 * fix something and we should rerun fsck tool again. 3305 * Display the full report from fsck. This displays all live and dead 3306 * region servers, and all known regions. 3307 */ 3308 void setShouldRerun() { 3309 rerun = true; 3310 } 3311 3312 public boolean shouldRerun() { 3313 return rerun; 3314 } 3315 3316 /** 3317 * Fix inconsistencies found by fsck. This should try to fix errors (if any) 3318 * found by fsck utility. 3319 */ 3320 public void setFixAssignments(boolean shouldFix) { 3321 fixAssignments = shouldFix; 3322 fixAny |= shouldFix; 3323 } 3324 3325 boolean shouldFixAssignments() { 3326 return fixAssignments; 3327 } 3328 3329 public void setFixMeta(boolean shouldFix) { 3330 fixMeta = shouldFix; 3331 fixAny |= shouldFix; 3332 } 3333 3334 boolean shouldFixMeta() { 3335 return fixMeta; 3336 } 3337 3338 public void setFixEmptyMetaCells(boolean shouldFix) { 3339 fixEmptyMetaCells = shouldFix; 3340 fixAny |= shouldFix; 3341 } 3342 3343 boolean shouldFixEmptyMetaCells() { 3344 return fixEmptyMetaCells; 3345 } 3346 3347 public void setCheckHdfs(boolean checking) { 3348 checkHdfs = checking; 3349 } 3350 3351 boolean shouldCheckHdfs() { 3352 return checkHdfs; 3353 } 3354 3355 public void setFixHdfsHoles(boolean shouldFix) { 3356 fixHdfsHoles = shouldFix; 3357 fixAny |= shouldFix; 3358 } 3359 3360 boolean shouldFixHdfsHoles() { 3361 return fixHdfsHoles; 3362 } 3363 3364 public void setFixTableOrphans(boolean shouldFix) { 3365 fixTableOrphans = shouldFix; 3366 fixAny |= shouldFix; 3367 } 3368 3369 boolean shouldFixTableOrphans() { 3370 return fixTableOrphans; 3371 } 3372 3373 public void setFixHdfsOverlaps(boolean shouldFix) { 3374 fixHdfsOverlaps = shouldFix; 3375 fixAny |= shouldFix; 3376 } 3377 3378 boolean shouldFixHdfsOverlaps() { 3379 return fixHdfsOverlaps; 3380 } 3381 3382 public void setFixHdfsOrphans(boolean shouldFix) { 3383 fixHdfsOrphans = shouldFix; 3384 fixAny |= shouldFix; 3385 } 3386 3387 boolean shouldFixHdfsOrphans() { 3388 return fixHdfsOrphans; 3389 } 3390 3391 public void setFixVersionFile(boolean shouldFix) { 3392 fixVersionFile = shouldFix; 3393 fixAny |= shouldFix; 3394 } 3395 3396 public boolean shouldFixVersionFile() { 3397 return fixVersionFile; 3398 } 3399 3400 public void setSidelineBigOverlaps(boolean sbo) { 3401 this.sidelineBigOverlaps = sbo; 3402 } 3403 3404 public boolean shouldSidelineBigOverlaps() { 3405 return sidelineBigOverlaps; 3406 } 3407 3408 public void setFixSplitParents(boolean shouldFix) { 3409 fixSplitParents = shouldFix; 3410 fixAny |= shouldFix; 3411 } 3412 3413 public void setRemoveParents(boolean shouldFix) { 3414 removeParents = shouldFix; 3415 fixAny |= shouldFix; 3416 } 3417 3418 boolean shouldFixSplitParents() { 3419 return fixSplitParents; 3420 } 3421 3422 boolean shouldRemoveParents() { 3423 return removeParents; 3424 } 3425 3426 public void setFixReferenceFiles(boolean shouldFix) { 3427 fixReferenceFiles = shouldFix; 3428 fixAny |= shouldFix; 3429 } 3430 3431 boolean shouldFixReferenceFiles() { 3432 return fixReferenceFiles; 3433 } 3434 3435 public void setFixHFileLinks(boolean shouldFix) { 3436 fixHFileLinks = shouldFix; 3437 fixAny |= shouldFix; 3438 } 3439 3440 boolean shouldFixHFileLinks() { 3441 return fixHFileLinks; 3442 } 3443 3444 public boolean shouldIgnorePreCheckPermission() { 3445 return !fixAny || ignorePreCheckPermission; 3446 } 3447 3448 public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) { 3449 this.ignorePreCheckPermission = ignorePreCheckPermission; 3450 } 3451 3452 /** 3453 * @param mm maximum number of regions to merge into a single region. 3454 */ 3455 public void setMaxMerge(int mm) { 3456 this.maxMerge = mm; 3457 } 3458 3459 public int getMaxMerge() { 3460 return maxMerge; 3461 } 3462 3463 public void setMaxOverlapsToSideline(int mo) { 3464 this.maxOverlapsToSideline = mo; 3465 } 3466 3467 public int getMaxOverlapsToSideline() { 3468 return maxOverlapsToSideline; 3469 } 3470 3471 /** 3472 * Only check/fix tables specified by the list, 3473 * Empty list means all tables are included. 3474 */ 3475 boolean isTableIncluded(TableName table) { 3476 return (tablesIncluded.isEmpty()) || tablesIncluded.contains(table); 3477 } 3478 3479 public void includeTable(TableName table) { 3480 tablesIncluded.add(table); 3481 } 3482 3483 Set<TableName> getIncludedTables() { 3484 return new HashSet<>(tablesIncluded); 3485 } 3486 3487 /** 3488 * We are interested in only those tables that have not changed their state in 3489 * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag 3490 * @param seconds - the time in seconds 3491 */ 3492 public void setTimeLag(long seconds) { 3493 timelag = seconds * 1000; // convert to milliseconds 3494 } 3495 3496 /** 3497 * 3498 * @param sidelineDir - HDFS path to sideline data 3499 */ 3500 public void setSidelineDir(String sidelineDir) { 3501 this.sidelineDir = new Path(sidelineDir); 3502 } 3503 3504 protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException { 3505 return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles); 3506 } 3507 3508 public HFileCorruptionChecker getHFilecorruptionChecker() { 3509 return hfcc; 3510 } 3511 3512 public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) { 3513 this.hfcc = hfcc; 3514 } 3515 3516 public void setRetCode(int code) { 3517 this.retcode = code; 3518 } 3519 3520 public int getRetCode() { 3521 return retcode; 3522 } 3523 3524 protected HBaseFsck printUsageAndExit() { 3525 StringWriter sw = new StringWriter(2048); 3526 PrintWriter out = new PrintWriter(sw); 3527 out.println(""); 3528 out.println("-----------------------------------------------------------------------"); 3529 out.println("NOTE: As of HBase version 2.0, the hbck tool is significantly changed."); 3530 out.println("In general, all Read-Only options are supported and can be be used"); 3531 out.println("safely. Most -fix/ -repair options are NOT supported. Please see usage"); 3532 out.println("below for details on which options are not supported."); 3533 out.println("-----------------------------------------------------------------------"); 3534 out.println(""); 3535 out.println("Usage: fsck [opts] {only tables}"); 3536 out.println(" where [opts] are:"); 3537 out.println(" -help Display help options (this)"); 3538 out.println(" -details Display full report of all regions."); 3539 out.println(" -timelag <timeInSeconds> Process only regions that " + 3540 " have not experienced any metadata updates in the last " + 3541 " <timeInSeconds> seconds."); 3542 out.println(" -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" + 3543 " before checking if the fix worked if run with -fix"); 3544 out.println(" -summary Print only summary of the tables and status."); 3545 out.println(" -metaonly Only check the state of the hbase:meta table."); 3546 out.println(" -sidelineDir <hdfs://> HDFS path to backup existing meta."); 3547 out.println(" -boundaries Verify that regions boundaries are the same between META and store files."); 3548 out.println(" -exclusive Abort if another hbck is exclusive or fixing."); 3549 3550 out.println(""); 3551 out.println(" Datafile Repair options: (expert features, use with caution!)"); 3552 out.println(" -checkCorruptHFiles Check all Hfiles by opening them to make sure they are valid"); 3553 out.println(" -sidelineCorruptHFiles Quarantine corrupted HFiles. implies -checkCorruptHFiles"); 3554 3555 out.println(""); 3556 out.println(" Replication options"); 3557 out.println(" -fixReplication Deletes replication queues for removed peers"); 3558 3559 out.println(""); 3560 out.println(" Metadata Repair options supported as of version 2.0: (expert features, use with caution!)"); 3561 out.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs."); 3562 out.println(" -fixReferenceFiles Try to offline lingering reference store files"); 3563 out.println(" -fixHFileLinks Try to offline lingering HFileLinks"); 3564 out.println(" -noHdfsChecking Don't load/check region info from HDFS." 3565 + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap"); 3566 out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check"); 3567 3568 out.println(""); 3569 out.println("NOTE: Following options are NOT supported as of HBase version 2.0+."); 3570 out.println(""); 3571 out.println(" UNSUPPORTED Metadata Repair options: (expert features, use with caution!)"); 3572 out.println(" -fix Try to fix region assignments. This is for backwards compatiblity"); 3573 out.println(" -fixAssignments Try to fix region assignments. Replaces the old -fix"); 3574 out.println(" -fixMeta Try to fix meta problems. This assumes HDFS region info is good."); 3575 out.println(" -fixHdfsHoles Try to fix region holes in hdfs."); 3576 out.println(" -fixHdfsOrphans Try to fix region dirs with no .regioninfo file in hdfs"); 3577 out.println(" -fixTableOrphans Try to fix table dirs with no .tableinfo file in hdfs (online mode only)"); 3578 out.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs."); 3579 out.println(" -maxMerge <n> When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)"); 3580 out.println(" -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps"); 3581 out.println(" -maxOverlapsToSideline <n> When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)"); 3582 out.println(" -fixSplitParents Try to force offline split parents to be online."); 3583 out.println(" -removeParents Try to offline and sideline lingering parents and keep daughter regions."); 3584 out.println(" -fixEmptyMetaCells Try to fix hbase:meta entries not referencing any region" 3585 + " (empty REGIONINFO_QUALIFIER rows)"); 3586 3587 out.println(""); 3588 out.println(" UNSUPPORTED Metadata Repair shortcuts"); 3589 out.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " + 3590 "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles" + 3591 "-fixHFileLinks"); 3592 out.println(" -repairHoles Shortcut for -fixAssignments -fixMeta -fixHdfsHoles"); 3593 out.println(""); 3594 out.println(" Replication options"); 3595 out.println(" -fixReplication Deletes replication queues for removed peers"); 3596 out.println(" -cleanReplicationBrarier [tableName] clean the replication barriers " + 3597 "of a specified table, tableName is required"); 3598 out.flush(); 3599 errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString()); 3600 3601 setRetCode(-2); 3602 return this; 3603 } 3604 3605 /** 3606 * Main program 3607 * 3608 * @param args 3609 * @throws Exception 3610 */ 3611 public static void main(String[] args) throws Exception { 3612 // create a fsck object 3613 Configuration conf = HBaseConfiguration.create(); 3614 Path hbasedir = FSUtils.getRootDir(conf); 3615 URI defaultFs = hbasedir.getFileSystem(conf).getUri(); 3616 FSUtils.setFsDefault(conf, new Path(defaultFs)); 3617 int ret = ToolRunner.run(new HBaseFsckTool(conf), args); 3618 System.exit(ret); 3619 } 3620 3621 /** 3622 * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line. 3623 */ 3624 static class HBaseFsckTool extends Configured implements Tool { 3625 HBaseFsckTool(Configuration conf) { super(conf); } 3626 @Override 3627 public int run(String[] args) throws Exception { 3628 HBaseFsck hbck = new HBaseFsck(getConf()); 3629 hbck.exec(hbck.executor, args); 3630 hbck.close(); 3631 return hbck.getRetCode(); 3632 } 3633 }; 3634 3635 3636 public HBaseFsck exec(ExecutorService exec, String[] args) 3637 throws KeeperException, IOException, InterruptedException, ReplicationException { 3638 long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN; 3639 3640 boolean checkCorruptHFiles = false; 3641 boolean sidelineCorruptHFiles = false; 3642 3643 // Process command-line args. 3644 for (int i = 0; i < args.length; i++) { 3645 String cmd = args[i]; 3646 if (cmd.equals("-help") || cmd.equals("-h")) { 3647 return printUsageAndExit(); 3648 } else if (cmd.equals("-details")) { 3649 setDisplayFullReport(); 3650 } else if (cmd.equals("-exclusive")) { 3651 setForceExclusive(); 3652 } else if (cmd.equals("-timelag")) { 3653 if (i == args.length - 1) { 3654 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value."); 3655 return printUsageAndExit(); 3656 } 3657 try { 3658 long timelag = Long.parseLong(args[++i]); 3659 setTimeLag(timelag); 3660 } catch (NumberFormatException e) { 3661 errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value."); 3662 return printUsageAndExit(); 3663 } 3664 } else if (cmd.equals("-sleepBeforeRerun")) { 3665 if (i == args.length - 1) { 3666 errors.reportError(ERROR_CODE.WRONG_USAGE, 3667 "HBaseFsck: -sleepBeforeRerun needs a value."); 3668 return printUsageAndExit(); 3669 } 3670 try { 3671 sleepBeforeRerun = Long.parseLong(args[++i]); 3672 } catch (NumberFormatException e) { 3673 errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value."); 3674 return printUsageAndExit(); 3675 } 3676 } else if (cmd.equals("-sidelineDir")) { 3677 if (i == args.length - 1) { 3678 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value."); 3679 return printUsageAndExit(); 3680 } 3681 setSidelineDir(args[++i]); 3682 } else if (cmd.equals("-fix")) { 3683 errors.reportError(ERROR_CODE.WRONG_USAGE, 3684 "This option is deprecated, please use -fixAssignments instead."); 3685 setFixAssignments(true); 3686 } else if (cmd.equals("-fixAssignments")) { 3687 setFixAssignments(true); 3688 } else if (cmd.equals("-fixMeta")) { 3689 setFixMeta(true); 3690 } else if (cmd.equals("-noHdfsChecking")) { 3691 setCheckHdfs(false); 3692 } else if (cmd.equals("-fixHdfsHoles")) { 3693 setFixHdfsHoles(true); 3694 } else if (cmd.equals("-fixHdfsOrphans")) { 3695 setFixHdfsOrphans(true); 3696 } else if (cmd.equals("-fixTableOrphans")) { 3697 setFixTableOrphans(true); 3698 } else if (cmd.equals("-fixHdfsOverlaps")) { 3699 setFixHdfsOverlaps(true); 3700 } else if (cmd.equals("-fixVersionFile")) { 3701 setFixVersionFile(true); 3702 } else if (cmd.equals("-sidelineBigOverlaps")) { 3703 setSidelineBigOverlaps(true); 3704 } else if (cmd.equals("-fixSplitParents")) { 3705 setFixSplitParents(true); 3706 } else if (cmd.equals("-removeParents")) { 3707 setRemoveParents(true); 3708 } else if (cmd.equals("-ignorePreCheckPermission")) { 3709 setIgnorePreCheckPermission(true); 3710 } else if (cmd.equals("-checkCorruptHFiles")) { 3711 checkCorruptHFiles = true; 3712 } else if (cmd.equals("-sidelineCorruptHFiles")) { 3713 sidelineCorruptHFiles = true; 3714 } else if (cmd.equals("-fixReferenceFiles")) { 3715 setFixReferenceFiles(true); 3716 } else if (cmd.equals("-fixHFileLinks")) { 3717 setFixHFileLinks(true); 3718 } else if (cmd.equals("-fixEmptyMetaCells")) { 3719 setFixEmptyMetaCells(true); 3720 } else if (cmd.equals("-repair")) { 3721 // this attempts to merge overlapping hdfs regions, needs testing 3722 // under load 3723 setFixHdfsHoles(true); 3724 setFixHdfsOrphans(true); 3725 setFixMeta(true); 3726 setFixAssignments(true); 3727 setFixHdfsOverlaps(true); 3728 setFixVersionFile(true); 3729 setSidelineBigOverlaps(true); 3730 setFixSplitParents(false); 3731 setCheckHdfs(true); 3732 setFixReferenceFiles(true); 3733 setFixHFileLinks(true); 3734 } else if (cmd.equals("-repairHoles")) { 3735 // this will make all missing hdfs regions available but may lose data 3736 setFixHdfsHoles(true); 3737 setFixHdfsOrphans(false); 3738 setFixMeta(true); 3739 setFixAssignments(true); 3740 setFixHdfsOverlaps(false); 3741 setSidelineBigOverlaps(false); 3742 setFixSplitParents(false); 3743 setCheckHdfs(true); 3744 } else if (cmd.equals("-maxOverlapsToSideline")) { 3745 if (i == args.length - 1) { 3746 errors.reportError(ERROR_CODE.WRONG_USAGE, 3747 "-maxOverlapsToSideline needs a numeric value argument."); 3748 return printUsageAndExit(); 3749 } 3750 try { 3751 int maxOverlapsToSideline = Integer.parseInt(args[++i]); 3752 setMaxOverlapsToSideline(maxOverlapsToSideline); 3753 } catch (NumberFormatException e) { 3754 errors.reportError(ERROR_CODE.WRONG_USAGE, 3755 "-maxOverlapsToSideline needs a numeric value argument."); 3756 return printUsageAndExit(); 3757 } 3758 } else if (cmd.equals("-maxMerge")) { 3759 if (i == args.length - 1) { 3760 errors.reportError(ERROR_CODE.WRONG_USAGE, 3761 "-maxMerge needs a numeric value argument."); 3762 return printUsageAndExit(); 3763 } 3764 try { 3765 int maxMerge = Integer.parseInt(args[++i]); 3766 setMaxMerge(maxMerge); 3767 } catch (NumberFormatException e) { 3768 errors.reportError(ERROR_CODE.WRONG_USAGE, 3769 "-maxMerge needs a numeric value argument."); 3770 return printUsageAndExit(); 3771 } 3772 } else if (cmd.equals("-summary")) { 3773 setSummary(); 3774 } else if (cmd.equals("-metaonly")) { 3775 setCheckMetaOnly(); 3776 } else if (cmd.equals("-boundaries")) { 3777 setRegionBoundariesCheck(); 3778 } else if (cmd.equals("-fixReplication")) { 3779 setFixReplication(true); 3780 } else if (cmd.equals("-cleanReplicationBarrier")) { 3781 setCleanReplicationBarrier(true); 3782 if(args[++i].startsWith("-")){ 3783 printUsageAndExit(); 3784 } 3785 setCleanReplicationBarrierTable(args[i]); 3786 } else if (cmd.startsWith("-")) { 3787 errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd); 3788 return printUsageAndExit(); 3789 } else { 3790 includeTable(TableName.valueOf(cmd)); 3791 errors.print("Allow checking/fixes for table: " + cmd); 3792 } 3793 } 3794 3795 errors.print("HBaseFsck command line options: " + StringUtils.join(args, " ")); 3796 3797 // pre-check current user has FS write permission or not 3798 try { 3799 preCheckPermission(); 3800 } catch (AccessDeniedException ace) { 3801 Runtime.getRuntime().exit(-1); 3802 } catch (IOException ioe) { 3803 Runtime.getRuntime().exit(-1); 3804 } 3805 3806 // do the real work of hbck 3807 connect(); 3808 3809 // after connecting to server above, we have server version 3810 // check if unsupported option is specified based on server version 3811 if (!isOptionsSupported(args)) { 3812 return printUsageAndExit(); 3813 } 3814 3815 try { 3816 // if corrupt file mode is on, first fix them since they may be opened later 3817 if (checkCorruptHFiles || sidelineCorruptHFiles) { 3818 LOG.info("Checking all hfiles for corruption"); 3819 HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles); 3820 setHFileCorruptionChecker(hfcc); // so we can get result 3821 Collection<TableName> tables = getIncludedTables(); 3822 Collection<Path> tableDirs = new ArrayList<>(); 3823 Path rootdir = FSUtils.getRootDir(getConf()); 3824 if (tables.size() > 0) { 3825 for (TableName t : tables) { 3826 tableDirs.add(FSUtils.getTableDir(rootdir, t)); 3827 } 3828 } else { 3829 tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir); 3830 } 3831 hfcc.checkTables(tableDirs); 3832 hfcc.report(errors); 3833 } 3834 3835 // check and fix table integrity, region consistency. 3836 int code = onlineHbck(); 3837 setRetCode(code); 3838 // If we have changed the HBase state it is better to run hbck again 3839 // to see if we haven't broken something else in the process. 3840 // We run it only once more because otherwise we can easily fall into 3841 // an infinite loop. 3842 if (shouldRerun()) { 3843 try { 3844 LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix..."); 3845 Thread.sleep(sleepBeforeRerun); 3846 } catch (InterruptedException ie) { 3847 LOG.warn("Interrupted while sleeping"); 3848 return this; 3849 } 3850 // Just report 3851 setFixAssignments(false); 3852 setFixMeta(false); 3853 setFixHdfsHoles(false); 3854 setFixHdfsOverlaps(false); 3855 setFixVersionFile(false); 3856 setFixTableOrphans(false); 3857 errors.resetErrors(); 3858 code = onlineHbck(); 3859 setRetCode(code); 3860 } 3861 } finally { 3862 IOUtils.closeQuietly(this); 3863 } 3864 return this; 3865 } 3866 3867 private boolean isOptionsSupported(String[] args) { 3868 boolean result = true; 3869 String hbaseServerVersion = status.getHBaseVersion(); 3870 if (VersionInfo.compareVersion("2.any.any", hbaseServerVersion) < 0) { 3871 // Process command-line args. 3872 for (String arg : args) { 3873 if (unsupportedOptionsInV2.contains(arg)) { 3874 errors.reportError(ERROR_CODE.UNSUPPORTED_OPTION, 3875 "option '" + arg + "' is not " + "supportted!"); 3876 result = false; 3877 break; 3878 } 3879 } 3880 } 3881 return result; 3882 } 3883 3884 public void setCleanReplicationBarrierTable(String cleanReplicationBarrierTable) { 3885 this.cleanReplicationBarrierTable = TableName.valueOf(cleanReplicationBarrierTable); 3886 } 3887 3888 public void cleanReplicationBarrier() throws IOException { 3889 if (!cleanReplicationBarrier || cleanReplicationBarrierTable == null) { 3890 return; 3891 } 3892 if (cleanReplicationBarrierTable.isSystemTable()) { 3893 errors.reportError(ERROR_CODE.INVALID_TABLE, 3894 "invalid table: " + cleanReplicationBarrierTable); 3895 return; 3896 } 3897 3898 boolean isGlobalScope = false; 3899 try { 3900 isGlobalScope = admin.getDescriptor(cleanReplicationBarrierTable).hasGlobalReplicationScope(); 3901 } catch (TableNotFoundException e) { 3902 LOG.info("we may need to clean some erroneous data due to bugs"); 3903 } 3904 3905 if (isGlobalScope) { 3906 errors.reportError(ERROR_CODE.INVALID_TABLE, 3907 "table's replication scope is global: " + cleanReplicationBarrierTable); 3908 return; 3909 } 3910 List<byte[]> regionNames = new ArrayList<>(); 3911 Scan barrierScan = new Scan(); 3912 barrierScan.setCaching(100); 3913 barrierScan.addFamily(HConstants.REPLICATION_BARRIER_FAMILY); 3914 barrierScan 3915 .withStartRow(MetaTableAccessor.getTableStartRowForMeta(cleanReplicationBarrierTable, 3916 MetaTableAccessor.QueryType.REGION)) 3917 .withStopRow(MetaTableAccessor.getTableStopRowForMeta(cleanReplicationBarrierTable, 3918 MetaTableAccessor.QueryType.REGION)); 3919 Result result; 3920 try (ResultScanner scanner = meta.getScanner(barrierScan)) { 3921 while ((result = scanner.next()) != null) { 3922 regionNames.add(result.getRow()); 3923 } 3924 } 3925 if (regionNames.size() <= 0) { 3926 errors.reportError(ERROR_CODE.INVALID_TABLE, 3927 "there is no barriers of this table: " + cleanReplicationBarrierTable); 3928 return; 3929 } 3930 ReplicationQueueStorage queueStorage = 3931 ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf()); 3932 List<ReplicationPeerDescription> peerDescriptions = admin.listReplicationPeers(); 3933 if (peerDescriptions != null && peerDescriptions.size() > 0) { 3934 List<String> peers = peerDescriptions.stream() 3935 .filter(peerConfig -> peerConfig.getPeerConfig() 3936 .needToReplicate(cleanReplicationBarrierTable)) 3937 .map(peerConfig -> peerConfig.getPeerId()).collect(Collectors.toList()); 3938 try { 3939 List<String> batch = new ArrayList<>(); 3940 for (String peer : peers) { 3941 for (byte[] regionName : regionNames) { 3942 batch.add(RegionInfo.encodeRegionName(regionName)); 3943 if (batch.size() % 100 == 0) { 3944 queueStorage.removeLastSequenceIds(peer, batch); 3945 batch.clear(); 3946 } 3947 } 3948 if (batch.size() > 0) { 3949 queueStorage.removeLastSequenceIds(peer, batch); 3950 batch.clear(); 3951 } 3952 } 3953 } catch (ReplicationException re) { 3954 throw new IOException(re); 3955 } 3956 } 3957 for (byte[] regionName : regionNames) { 3958 meta.delete(new Delete(regionName).addFamily(HConstants.REPLICATION_BARRIER_FAMILY)); 3959 } 3960 setShouldRerun(); 3961 } 3962 3963 /** 3964 * ls -r for debugging purposes 3965 */ 3966 void debugLsr(Path p) throws IOException { 3967 debugLsr(getConf(), p, errors); 3968 } 3969 3970 /** 3971 * ls -r for debugging purposes 3972 */ 3973 public static void debugLsr(Configuration conf, 3974 Path p) throws IOException { 3975 debugLsr(conf, p, new PrintingErrorReporter()); 3976 } 3977 3978 /** 3979 * ls -r for debugging purposes 3980 */ 3981 public static void debugLsr(Configuration conf, 3982 Path p, HbckErrorReporter errors) throws IOException { 3983 if (!LOG.isDebugEnabled() || p == null) { 3984 return; 3985 } 3986 FileSystem fs = p.getFileSystem(conf); 3987 3988 if (!fs.exists(p)) { 3989 // nothing 3990 return; 3991 } 3992 errors.print(p.toString()); 3993 3994 if (fs.isFile(p)) { 3995 return; 3996 } 3997 3998 if (fs.getFileStatus(p).isDirectory()) { 3999 FileStatus[] fss= fs.listStatus(p); 4000 for (FileStatus status : fss) { 4001 debugLsr(conf, status.getPath(), errors); 4002 } 4003 } 4004 } 4005}