001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.util; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.Collection; 023import java.util.HashMap; 024import java.util.HashSet; 025import java.util.List; 026import java.util.Map; 027import java.util.Set; 028import java.util.SortedSet; 029import java.util.TreeMap; 030import java.util.TreeSet; 031import java.util.concurrent.ExecutionException; 032import java.util.concurrent.Future; 033import org.apache.hadoop.conf.Configuration; 034import org.apache.hadoop.fs.FileSystem; 035import org.apache.hadoop.fs.Path; 036import org.apache.hadoop.hbase.HConstants; 037import org.apache.hadoop.hbase.ServerName; 038import org.apache.hadoop.hbase.TableName; 039import org.apache.hadoop.hbase.client.RegionInfo; 040import org.apache.hadoop.hbase.client.RegionInfoBuilder; 041import org.apache.hadoop.hbase.client.TableDescriptor; 042import org.apache.hadoop.hbase.regionserver.HRegion; 043import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool; 044import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler; 045import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl; 046import org.apache.yetus.audience.InterfaceAudience; 047import org.apache.yetus.audience.InterfaceStability; 048import org.slf4j.Logger; 049import org.slf4j.LoggerFactory; 050 051import org.apache.hbase.thirdparty.com.google.common.base.Joiner; 052import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; 053import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; 054import org.apache.hbase.thirdparty.com.google.common.collect.Multimap; 055import org.apache.hbase.thirdparty.com.google.common.collect.Ordering; 056import org.apache.hbase.thirdparty.com.google.common.collect.TreeMultimap; 057 058/** 059 * Maintain information about a particular table. 060 */ 061@InterfaceAudience.Private 062@InterfaceStability.Evolving 063public class HbckTableInfo { 064 private static final Logger LOG = LoggerFactory.getLogger(HbckTableInfo.class.getName()); 065 066 private static final String TO_BE_LOADED = "to_be_loaded"; 067 068 TableName tableName; 069 TreeSet<ServerName> deployedOn; 070 071 // backwards regions 072 final List<HbckRegionInfo> backwards = new ArrayList<>(); 073 074 // sidelined big overlapped regions 075 final Map<Path, HbckRegionInfo> sidelinedRegions = new HashMap<>(); 076 077 // region split calculator 078 final RegionSplitCalculator<HbckRegionInfo> sc = 079 new RegionSplitCalculator<>(HbckRegionInfo.COMPARATOR); 080 081 // Histogram of different TableDescriptors found. Ideally there is only one! 082 final Set<TableDescriptor> htds = new HashSet<>(); 083 084 // key = start split, values = set of splits in problem group 085 final Multimap<byte[], HbckRegionInfo> overlapGroups = 086 TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, HbckRegionInfo.COMPARATOR); 087 088 // list of regions derived from meta entries. 089 private ImmutableList<RegionInfo> regionsFromMeta = null; 090 091 HBaseFsck hbck; 092 093 HbckTableInfo(TableName name, HBaseFsck hbck) { 094 this.tableName = name; 095 this.hbck = hbck; 096 deployedOn = new TreeSet<>(); 097 } 098 099 /** 100 * @return descriptor common to all regions. null if are none or multiple! 101 */ 102 TableDescriptor getTableDescriptor() { 103 if (htds.size() == 1) { 104 return (TableDescriptor) htds.toArray()[0]; 105 } else { 106 LOG.error( 107 "None/Multiple table descriptors found for table '" + tableName + "' regions: " + htds); 108 } 109 return null; 110 } 111 112 public void addRegionInfo(HbckRegionInfo hir) { 113 if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) { 114 // end key is absolute end key, just add it. 115 // ignore replicas other than primary for these checks 116 if (hir.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { 117 sc.add(hir); 118 } 119 return; 120 } 121 122 // if not the absolute end key, check for cycle 123 if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) { 124 hbck.getErrors().reportError(HbckErrorReporter.ERROR_CODE.REGION_CYCLE, 125 String.format( 126 "The endkey for this region comes before the " + "startkey, startkey=%s, endkey=%s", 127 Bytes.toStringBinary(hir.getStartKey()), Bytes.toStringBinary(hir.getEndKey())), 128 this, hir); 129 backwards.add(hir); 130 return; 131 } 132 133 // main case, add to split calculator 134 // ignore replicas other than primary for these checks 135 if (hir.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { 136 sc.add(hir); 137 } 138 } 139 140 public void addServer(ServerName server) { 141 this.deployedOn.add(server); 142 } 143 144 public TableName getName() { 145 return tableName; 146 } 147 148 public int getNumRegions() { 149 return sc.getStarts().size() + backwards.size(); 150 } 151 152 public synchronized ImmutableList<RegionInfo> 153 getRegionsFromMeta(TreeMap<String, HbckRegionInfo> regionInfoMap) { 154 // lazy loaded, synchronized to ensure a single load 155 if (regionsFromMeta == null) { 156 List<RegionInfo> regions = new ArrayList<>(); 157 for (HbckRegionInfo h : regionInfoMap.values()) { 158 if (tableName.equals(h.getTableName())) { 159 if (h.getMetaEntry() != null) { 160 regions.add(h.getMetaEntry().getRegionInfo()); 161 } 162 } 163 } 164 regionsFromMeta = Ordering.from(RegionInfo.COMPARATOR).immutableSortedCopy(regions); 165 } 166 167 return regionsFromMeta; 168 } 169 170 class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl { 171 HbckErrorReporter errors; 172 173 IntegrityFixSuggester(HbckTableInfo ti, HbckErrorReporter errors) { 174 this.errors = errors; 175 setTableInfo(ti); 176 } 177 178 @Override 179 public void handleRegionStartKeyNotEmpty(HbckRegionInfo hi) throws IOException { 180 errors.reportError(HbckErrorReporter.ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY, 181 "First region should start with an empty key. You need to " 182 + " create a new region and regioninfo in HDFS to plug the hole.", 183 getTableInfo(), hi); 184 } 185 186 @Override 187 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException { 188 errors.reportError(HbckErrorReporter.ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY, 189 "Last region should end with an empty key. You need to " 190 + "create a new region and regioninfo in HDFS to plug the hole.", 191 getTableInfo()); 192 } 193 194 @Override 195 public void handleDegenerateRegion(HbckRegionInfo hi) throws IOException { 196 errors.reportError(HbckErrorReporter.ERROR_CODE.DEGENERATE_REGION, 197 "Region has the same start and end key.", getTableInfo(), hi); 198 } 199 200 @Override 201 public void handleDuplicateStartKeys(HbckRegionInfo r1, HbckRegionInfo r2) throws IOException { 202 byte[] key = r1.getStartKey(); 203 // dup start key 204 errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_STARTKEYS, 205 "Multiple regions have the same startkey: " + Bytes.toStringBinary(key), getTableInfo(), 206 r1); 207 errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_STARTKEYS, 208 "Multiple regions have the same startkey: " + Bytes.toStringBinary(key), getTableInfo(), 209 r2); 210 } 211 212 @Override 213 public void handleSplit(HbckRegionInfo r1, HbckRegionInfo r2) throws IOException { 214 byte[] key = r1.getStartKey(); 215 // dup start key 216 errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_ENDKEYS, 217 "Multiple regions have the same regionID: " + Bytes.toStringBinary(key), getTableInfo(), 218 r1); 219 errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_ENDKEYS, 220 "Multiple regions have the same regionID: " + Bytes.toStringBinary(key), getTableInfo(), 221 r2); 222 } 223 224 @Override 225 public void handleOverlapInRegionChain(HbckRegionInfo hi1, HbckRegionInfo hi2) 226 throws IOException { 227 errors.reportError(HbckErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN, 228 "There is an overlap in the region chain.", getTableInfo(), hi1, hi2); 229 } 230 231 @Override 232 public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException { 233 errors.reportError(HbckErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN, 234 "There is a hole in the region chain between " + Bytes.toStringBinary(holeStart) + " and " 235 + Bytes.toStringBinary(holeStop) + ". You need to create a new .regioninfo and region " 236 + "dir in hdfs to plug the hole."); 237 } 238 } 239 240 /** 241 * This handler fixes integrity errors from hdfs information. There are basically three classes of 242 * integrity problems 1) holes, 2) overlaps, and 3) invalid regions. This class overrides methods 243 * that fix holes and the overlap group case. Individual cases of particular overlaps are handled 244 * by the general overlap group merge repair case. If hbase is online, this forces regions offline 245 * before doing merge operations. 246 */ 247 class HDFSIntegrityFixer extends IntegrityFixSuggester { 248 Configuration conf; 249 250 boolean fixOverlaps = true; 251 252 HDFSIntegrityFixer(HbckTableInfo ti, HbckErrorReporter errors, Configuration conf, 253 boolean fixHoles, boolean fixOverlaps) { 254 super(ti, errors); 255 this.conf = conf; 256 this.fixOverlaps = fixOverlaps; 257 // TODO properly use fixHoles 258 } 259 260 /** 261 * This is a special case hole -- when the first region of a table is missing from META, HBase 262 * doesn't acknowledge the existance of the table. 263 */ 264 @Override 265 public void handleRegionStartKeyNotEmpty(HbckRegionInfo next) throws IOException { 266 errors.reportError(HbckErrorReporter.ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY, 267 "First region should start with an empty key. Creating a new " 268 + "region and regioninfo in HDFS to plug the hole.", 269 getTableInfo(), next); 270 TableDescriptor htd = getTableInfo().getTableDescriptor(); 271 // from special EMPTY_START_ROW to next region's startKey 272 RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName()) 273 .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(next.getStartKey()).build(); 274 275 // TODO test 276 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); 277 LOG.info("Table region start key was not empty. Created new empty region: " + newRegion + " " 278 + region); 279 hbck.fixes++; 280 } 281 282 @Override 283 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException { 284 errors.reportError(HbckErrorReporter.ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY, 285 "Last region should end with an empty key. Creating a new " 286 + "region and regioninfo in HDFS to plug the hole.", 287 getTableInfo()); 288 TableDescriptor htd = getTableInfo().getTableDescriptor(); 289 // from curEndKey to EMPTY_START_ROW 290 RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(curEndKey) 291 .setEndKey(HConstants.EMPTY_START_ROW).build(); 292 293 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); 294 LOG.info("Table region end key was not empty. Created new empty region: " + newRegion + " " 295 + region); 296 hbck.fixes++; 297 } 298 299 /** 300 * There is a hole in the hdfs regions that violates the table integrity rules. Create a new 301 * empty region that patches the hole. 302 */ 303 @Override 304 public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) 305 throws IOException { 306 errors.reportError(HbckErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN, 307 "There is a hole in the region chain between " + Bytes.toStringBinary(holeStartKey) 308 + " and " + Bytes.toStringBinary(holeStopKey) + ". Creating a new regioninfo and region " 309 + "dir in hdfs to plug the hole."); 310 TableDescriptor htd = getTableInfo().getTableDescriptor(); 311 RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName()) 312 .setStartKey(holeStartKey).setEndKey(holeStopKey).build(); 313 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); 314 LOG.info("Plugged hole by creating new empty region: " + newRegion + " " + region); 315 hbck.fixes++; 316 } 317 318 /** 319 * This takes set of overlapping regions and merges them into a single region. This covers cases 320 * like degenerate regions, shared start key, general overlaps, duplicate ranges, and partial 321 * overlapping regions. Cases: - Clean regions that overlap - Only .oldlogs regions (can't find 322 * start/stop range, or figure out) This is basically threadsafe, except for the fixer increment 323 * in mergeOverlaps. 324 */ 325 @Override 326 public void handleOverlapGroup(Collection<HbckRegionInfo> overlap) throws IOException { 327 Preconditions.checkNotNull(overlap); 328 Preconditions.checkArgument(overlap.size() > 0); 329 330 if (!this.fixOverlaps) { 331 LOG.warn("Not attempting to repair overlaps."); 332 return; 333 } 334 335 if (overlap.size() > hbck.getMaxMerge()) { 336 LOG.warn( 337 "Overlap group has " + overlap.size() + " overlapping " + "regions which is greater than " 338 + hbck.getMaxMerge() + ", the max number of regions to merge"); 339 if (hbck.shouldSidelineBigOverlaps()) { 340 // we only sideline big overlapped groups that exceeds the max number of regions to merge 341 sidelineBigOverlaps(overlap); 342 } 343 return; 344 } 345 if (hbck.shouldRemoveParents()) { 346 removeParentsAndFixSplits(overlap); 347 } 348 mergeOverlaps(overlap); 349 } 350 351 void removeParentsAndFixSplits(Collection<HbckRegionInfo> overlap) throws IOException { 352 Pair<byte[], byte[]> range = null; 353 HbckRegionInfo parent = null; 354 HbckRegionInfo daughterA = null; 355 HbckRegionInfo daughterB = null; 356 Collection<HbckRegionInfo> daughters = new ArrayList<HbckRegionInfo>(overlap); 357 358 String thread = Thread.currentThread().getName(); 359 LOG.info("== [" + thread + "] Attempting fix splits in overlap state."); 360 361 // we only can handle a single split per group at the time 362 if (overlap.size() > 3) { 363 LOG.info("Too many overlaps were found on this group, falling back to regular merge."); 364 return; 365 } 366 367 for (HbckRegionInfo hi : overlap) { 368 if (range == null) { 369 range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey()); 370 } else { 371 if ( 372 RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getStartKey(), range.getFirst()) < 0 373 ) { 374 range.setFirst(hi.getStartKey()); 375 } 376 if ( 377 RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getEndKey(), range.getSecond()) > 0 378 ) { 379 range.setSecond(hi.getEndKey()); 380 } 381 } 382 } 383 384 LOG.info("This group range is [" + Bytes.toStringBinary(range.getFirst()) + ", " 385 + Bytes.toStringBinary(range.getSecond()) + "]"); 386 387 // attempt to find a possible parent for the edge case of a split 388 for (HbckRegionInfo hi : overlap) { 389 if ( 390 Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0 391 && Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0 392 ) { 393 LOG.info("This is a parent for this group: " + hi.toString()); 394 parent = hi; 395 } 396 } 397 398 // Remove parent regions from daughters collection 399 if (parent != null) { 400 daughters.remove(parent); 401 } 402 403 // Lets verify that daughters share the regionID at split time and they 404 // were created after the parent 405 for (HbckRegionInfo hi : daughters) { 406 if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0) { 407 if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) { 408 daughterA = hi; 409 } 410 } 411 if (Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) { 412 if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) { 413 daughterB = hi; 414 } 415 } 416 } 417 418 // daughters must share the same regionID and we should have a parent too 419 if ( 420 daughterA.getHdfsHRI().getRegionId() != daughterB.getHdfsHRI().getRegionId() 421 || parent == null 422 ) { 423 return; 424 } 425 426 FileSystem fs = FileSystem.get(conf); 427 LOG.info("Found parent: " + parent.getRegionNameAsString()); 428 LOG.info("Found potential daughter a: " + daughterA.getRegionNameAsString()); 429 LOG.info("Found potential daughter b: " + daughterB.getRegionNameAsString()); 430 LOG.info("Trying to fix parent in overlap by removing the parent."); 431 try { 432 hbck.closeRegion(parent); 433 } catch (IOException ioe) { 434 LOG.warn("Parent region could not be closed, continuing with regular merge...", ioe); 435 return; 436 } catch (InterruptedException ie) { 437 LOG.warn("Parent region could not be closed, continuing with regular merge...", ie); 438 return; 439 } 440 441 try { 442 hbck.offline(parent.getRegionName()); 443 } catch (IOException ioe) { 444 LOG.warn("Unable to offline parent region: " + parent.getRegionNameAsString() 445 + ". Just continuing with regular merge... ", ioe); 446 return; 447 } 448 449 try { 450 HBaseFsckRepair.removeParentInMeta(conf, parent.getHdfsHRI()); 451 } catch (IOException ioe) { 452 LOG.warn("Unable to remove parent region in META: " + parent.getRegionNameAsString() 453 + ". Just continuing with regular merge... ", ioe); 454 return; 455 } 456 457 hbck.sidelineRegionDir(fs, parent); 458 LOG.info("[" + thread + "] Sidelined parent region dir " + parent.getHdfsRegionDir() 459 + " into " + hbck.getSidelineDir()); 460 hbck.debugLsr(parent.getHdfsRegionDir()); 461 462 // Make sure we don't have the parents and daughters around 463 overlap.remove(parent); 464 overlap.remove(daughterA); 465 overlap.remove(daughterB); 466 467 LOG.info("Done fixing split."); 468 469 } 470 471 void mergeOverlaps(Collection<HbckRegionInfo> overlap) throws IOException { 472 String thread = Thread.currentThread().getName(); 473 LOG.info( 474 "== [" + thread + "] Merging regions into one region: " + Joiner.on(",").join(overlap)); 475 // get the min / max range and close all concerned regions 476 Pair<byte[], byte[]> range = null; 477 for (HbckRegionInfo hi : overlap) { 478 if (range == null) { 479 range = new Pair<>(hi.getStartKey(), hi.getEndKey()); 480 } else { 481 if ( 482 RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getStartKey(), range.getFirst()) < 0 483 ) { 484 range.setFirst(hi.getStartKey()); 485 } 486 if ( 487 RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getEndKey(), range.getSecond()) > 0 488 ) { 489 range.setSecond(hi.getEndKey()); 490 } 491 } 492 // need to close files so delete can happen. 493 LOG.debug("[" + thread + "] Closing region before moving data around: " + hi); 494 LOG.debug("[" + thread + "] Contained region dir before close"); 495 hbck.debugLsr(hi.getHdfsRegionDir()); 496 try { 497 LOG.info("[" + thread + "] Closing region: " + hi); 498 hbck.closeRegion(hi); 499 } catch (IOException ioe) { 500 LOG.warn("[" + thread + "] Was unable to close region " + hi + ". Just continuing... ", 501 ioe); 502 } catch (InterruptedException e) { 503 LOG.warn("[" + thread + "] Was unable to close region " + hi + ". Just continuing... ", 504 e); 505 } 506 507 try { 508 LOG.info("[" + thread + "] Offlining region: " + hi); 509 hbck.offline(hi.getRegionName()); 510 } catch (IOException ioe) { 511 LOG.warn("[" + thread + "] Unable to offline region from master: " + hi 512 + ". Just continuing... ", ioe); 513 } 514 } 515 516 // create new empty container region. 517 TableDescriptor htd = getTableInfo().getTableDescriptor(); 518 // from start key to end Key 519 RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName()) 520 .setStartKey(range.getFirst()).setEndKey(range.getSecond()).build(); 521 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); 522 LOG.info("[" + thread + "] Created new empty container region: " + newRegion 523 + " to contain regions: " + Joiner.on(",").join(overlap)); 524 hbck.debugLsr(region.getRegionFileSystem().getRegionDir()); 525 526 // all target regions are closed, should be able to safely cleanup. 527 boolean didFix = false; 528 Path target = region.getRegionFileSystem().getRegionDir(); 529 for (HbckRegionInfo contained : overlap) { 530 LOG.info("[" + thread + "] Merging " + contained + " into " + target); 531 int merges = hbck.mergeRegionDirs(target, contained); 532 if (merges > 0) { 533 didFix = true; 534 } 535 } 536 if (didFix) { 537 hbck.fixes++; 538 } 539 } 540 541 /** 542 * Sideline some regions in a big overlap group so that it will have fewer regions, and it is 543 * easier to merge them later on. 544 * @param bigOverlap the overlapped group with regions more than maxMerge 545 */ 546 void sidelineBigOverlaps(Collection<HbckRegionInfo> bigOverlap) throws IOException { 547 int overlapsToSideline = bigOverlap.size() - hbck.getMaxMerge(); 548 if (overlapsToSideline > hbck.getMaxOverlapsToSideline()) { 549 overlapsToSideline = hbck.getMaxOverlapsToSideline(); 550 } 551 List<HbckRegionInfo> regionsToSideline = 552 RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline); 553 FileSystem fs = FileSystem.get(conf); 554 for (HbckRegionInfo regionToSideline : regionsToSideline) { 555 try { 556 LOG.info("Closing region: " + regionToSideline); 557 hbck.closeRegion(regionToSideline); 558 } catch (IOException ioe) { 559 LOG.warn("Was unable to close region " + regionToSideline + ". Just continuing... ", 560 ioe); 561 } catch (InterruptedException e) { 562 LOG.warn("Was unable to close region " + regionToSideline + ". Just continuing... ", e); 563 } 564 565 try { 566 LOG.info("Offlining region: " + regionToSideline); 567 hbck.offline(regionToSideline.getRegionName()); 568 } catch (IOException ioe) { 569 LOG.warn( 570 "Unable to offline region from master: " + regionToSideline + ". Just continuing... ", 571 ioe); 572 } 573 574 LOG.info("Before sideline big overlapped region: " + regionToSideline.toString()); 575 Path sidelineRegionDir = hbck.sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline); 576 if (sidelineRegionDir != null) { 577 sidelinedRegions.put(sidelineRegionDir, regionToSideline); 578 LOG.info("After sidelined big overlapped region: " 579 + regionToSideline.getRegionNameAsString() + " to " + sidelineRegionDir.toString()); 580 hbck.fixes++; 581 } 582 } 583 } 584 } 585 586 /** 587 * Check the region chain (from META) of this table. We are looking for holes, overlaps, and 588 * cycles. 589 * @return false if there are errors 590 */ 591 public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException { 592 // When table is disabled no need to check for the region chain. Some of the regions 593 // accidently if deployed, this below code might report some issues like missing start 594 // or end regions or region hole in chain and may try to fix which is unwanted. 595 if (hbck.isTableDisabled(this.tableName)) { 596 return true; 597 } 598 int originalErrorsCount = hbck.getErrors().getErrorList().size(); 599 Multimap<byte[], HbckRegionInfo> regions = sc.calcCoverage(); 600 SortedSet<byte[]> splits = sc.getSplits(); 601 602 byte[] prevKey = null; 603 byte[] problemKey = null; 604 605 if (splits.isEmpty()) { 606 // no region for this table 607 handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); 608 } 609 610 for (byte[] key : splits) { 611 Collection<HbckRegionInfo> ranges = regions.get(key); 612 if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) { 613 for (HbckRegionInfo rng : ranges) { 614 handler.handleRegionStartKeyNotEmpty(rng); 615 } 616 } 617 618 // check for degenerate ranges 619 for (HbckRegionInfo rng : ranges) { 620 // special endkey case converts '' to null 621 byte[] endKey = rng.getEndKey(); 622 endKey = (endKey.length == 0) ? null : endKey; 623 if (Bytes.equals(rng.getStartKey(), endKey)) { 624 handler.handleDegenerateRegion(rng); 625 } 626 } 627 628 if (ranges.size() == 1) { 629 // this split key is ok -- no overlap, not a hole. 630 if (problemKey != null) { 631 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key)); 632 } 633 problemKey = null; // fell through, no more problem. 634 } else if (ranges.size() > 1) { 635 // set the new problem key group name, if already have problem key, just 636 // keep using it. 637 if (problemKey == null) { 638 // only for overlap regions. 639 LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key)); 640 problemKey = key; 641 } 642 overlapGroups.putAll(problemKey, ranges); 643 644 // record errors 645 ArrayList<HbckRegionInfo> subRange = new ArrayList<>(ranges); 646 // this dumb and n^2 but this shouldn't happen often 647 for (HbckRegionInfo r1 : ranges) { 648 if (r1.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 649 continue; 650 } 651 subRange.remove(r1); 652 for (HbckRegionInfo r2 : subRange) { 653 if (r2.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { 654 continue; 655 } 656 // general case of same start key 657 if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey()) == 0) { 658 handler.handleDuplicateStartKeys(r1, r2); 659 } else if ( 660 Bytes.compareTo(r1.getEndKey(), r2.getStartKey()) == 0 661 && r1.getHdfsHRI().getRegionId() == r2.getHdfsHRI().getRegionId() 662 ) { 663 LOG.info("this is a split, log to splits"); 664 handler.handleSplit(r1, r2); 665 } else { 666 // overlap 667 handler.handleOverlapInRegionChain(r1, r2); 668 } 669 } 670 } 671 672 } else if (ranges.isEmpty()) { 673 if (problemKey != null) { 674 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key)); 675 } 676 problemKey = null; 677 678 byte[] holeStopKey = sc.getSplits().higher(key); 679 // if higher key is null we reached the top. 680 if (holeStopKey != null) { 681 // hole 682 handler.handleHoleInRegionChain(key, holeStopKey); 683 } 684 } 685 prevKey = key; 686 } 687 688 // When the last region of a table is proper and having an empty end key, 'prevKey' 689 // will be null. 690 if (prevKey != null) { 691 handler.handleRegionEndKeyNotEmpty(prevKey); 692 } 693 694 // TODO fold this into the TableIntegrityHandler 695 if (hbck.getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) { 696 boolean ok = handleOverlapsParallel(handler, prevKey); 697 if (!ok) { 698 return false; 699 } 700 } else { 701 for (Collection<HbckRegionInfo> overlap : overlapGroups.asMap().values()) { 702 handler.handleOverlapGroup(overlap); 703 } 704 } 705 706 if (HBaseFsck.shouldDisplayFullReport()) { 707 // do full region split map dump 708 hbck.getErrors().print("---- Table '" + this.tableName + "': region split map"); 709 dump(splits, regions); 710 hbck.getErrors().print("---- Table '" + this.tableName + "': overlap groups"); 711 dumpOverlapProblems(overlapGroups); 712 hbck.getErrors().print("There are " + overlapGroups.keySet().size() + " overlap groups with " 713 + overlapGroups.size() + " overlapping regions"); 714 } 715 if (!sidelinedRegions.isEmpty()) { 716 LOG.warn("Sidelined big overlapped regions, please bulk load them!"); 717 hbck.getErrors() 718 .print("---- Table '" + this.tableName + "': sidelined big overlapped regions"); 719 dumpSidelinedRegions(sidelinedRegions); 720 } 721 return hbck.getErrors().getErrorList().size() == originalErrorsCount; 722 } 723 724 private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey) 725 throws IOException { 726 // we parallelize overlap handler for the case we have lots of groups to fix. We can 727 // safely assume each group is independent. 728 List<HBaseFsck.WorkItemOverlapMerge> merges = new ArrayList<>(overlapGroups.size()); 729 List<Future<Void>> rets; 730 for (Collection<HbckRegionInfo> overlap : overlapGroups.asMap().values()) { 731 // 732 merges.add(new HBaseFsck.WorkItemOverlapMerge(overlap, handler)); 733 } 734 try { 735 rets = hbck.executor.invokeAll(merges); 736 } catch (InterruptedException e) { 737 LOG.error("Overlap merges were interrupted", e); 738 return false; 739 } 740 for (int i = 0; i < merges.size(); i++) { 741 HBaseFsck.WorkItemOverlapMerge work = merges.get(i); 742 Future<Void> f = rets.get(i); 743 try { 744 f.get(); 745 } catch (ExecutionException e) { 746 LOG.warn("Failed to merge overlap group" + work, e.getCause()); 747 } catch (InterruptedException e) { 748 LOG.error("Waiting for overlap merges was interrupted", e); 749 return false; 750 } 751 } 752 return true; 753 } 754 755 /** 756 * This dumps data in a visually reasonable way for visual debugging 757 */ 758 private void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckRegionInfo> regions) { 759 // we display this way because the last end key should be displayed as well. 760 StringBuilder sb = new StringBuilder(); 761 for (byte[] k : splits) { 762 sb.setLength(0); // clear out existing buffer, if any. 763 sb.append(Bytes.toStringBinary(k) + ":\t"); 764 for (HbckRegionInfo r : regions.get(k)) { 765 sb.append("[ " + r.toString() + ", " + Bytes.toStringBinary(r.getEndKey()) + "]\t"); 766 } 767 hbck.getErrors().print(sb.toString()); 768 } 769 } 770 771 private void dumpOverlapProblems(Multimap<byte[], HbckRegionInfo> regions) { 772 // we display this way because the last end key should be displayed as 773 // well. 774 for (byte[] k : regions.keySet()) { 775 hbck.getErrors().print(Bytes.toStringBinary(k) + ":"); 776 for (HbckRegionInfo r : regions.get(k)) { 777 hbck.getErrors() 778 .print("[ " + r.toString() + ", " + Bytes.toStringBinary(r.getEndKey()) + "]"); 779 } 780 hbck.getErrors().print("----"); 781 } 782 } 783 784 private void dumpSidelinedRegions(Map<Path, HbckRegionInfo> regions) { 785 for (Map.Entry<Path, HbckRegionInfo> entry : regions.entrySet()) { 786 TableName tableName = entry.getValue().getTableName(); 787 Path path = entry.getKey(); 788 hbck.getErrors().print("This sidelined region dir should be bulk loaded: " + path.toString()); 789 hbck.getErrors().print("Bulk load command looks like: " + BulkLoadHFilesTool.NAME + " " 790 + path.toUri().getPath() + " " + tableName); 791 } 792 } 793}