001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Collection;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.List;
026import java.util.Map;
027import java.util.Set;
028import java.util.SortedSet;
029import java.util.TreeMap;
030import java.util.TreeSet;
031import java.util.concurrent.ExecutionException;
032import java.util.concurrent.Future;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.ServerName;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.RegionInfoBuilder;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.regionserver.HRegion;
043import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool;
044import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
045import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
046import org.apache.yetus.audience.InterfaceAudience;
047import org.apache.yetus.audience.InterfaceStability;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
052import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
053import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
054import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
055import org.apache.hbase.thirdparty.com.google.common.collect.Ordering;
056import org.apache.hbase.thirdparty.com.google.common.collect.TreeMultimap;
057
058/**
059 * Maintain information about a particular table.
060 */
061@InterfaceAudience.Private
062@InterfaceStability.Evolving
063public class HbckTableInfo {
064  private static final Logger LOG = LoggerFactory.getLogger(HbckTableInfo.class.getName());
065
066  private static final String TO_BE_LOADED = "to_be_loaded";
067
068  TableName tableName;
069  TreeSet<ServerName> deployedOn;
070
071  // backwards regions
072  final List<HbckRegionInfo> backwards = new ArrayList<>();
073
074  // sidelined big overlapped regions
075  final Map<Path, HbckRegionInfo> sidelinedRegions = new HashMap<>();
076
077  // region split calculator
078  final RegionSplitCalculator<HbckRegionInfo> sc =
079    new RegionSplitCalculator<>(HbckRegionInfo.COMPARATOR);
080
081  // Histogram of different TableDescriptors found. Ideally there is only one!
082  final Set<TableDescriptor> htds = new HashSet<>();
083
084  // key = start split, values = set of splits in problem group
085  final Multimap<byte[], HbckRegionInfo> overlapGroups =
086    TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, HbckRegionInfo.COMPARATOR);
087
088  // list of regions derived from meta entries.
089  private ImmutableList<RegionInfo> regionsFromMeta = null;
090
091  HBaseFsck hbck;
092
093  HbckTableInfo(TableName name, HBaseFsck hbck) {
094    this.tableName = name;
095    this.hbck = hbck;
096    deployedOn = new TreeSet<>();
097  }
098
099  /** Returns descriptor common to all regions. null if are none or multiple! */
100  TableDescriptor getTableDescriptor() {
101    if (htds.size() == 1) {
102      return (TableDescriptor) htds.toArray()[0];
103    } else {
104      LOG.error(
105        "None/Multiple table descriptors found for table '" + tableName + "' regions: " + htds);
106    }
107    return null;
108  }
109
110  public void addRegionInfo(HbckRegionInfo hir) {
111    if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
112      // end key is absolute end key, just add it.
113      // ignore replicas other than primary for these checks
114      if (hir.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
115        sc.add(hir);
116      }
117      return;
118    }
119
120    // if not the absolute end key, check for cycle
121    if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
122      hbck.getErrors().reportError(HbckErrorReporter.ERROR_CODE.REGION_CYCLE,
123        String.format(
124          "The endkey for this region comes before the " + "startkey, startkey=%s, endkey=%s",
125          Bytes.toStringBinary(hir.getStartKey()), Bytes.toStringBinary(hir.getEndKey())),
126        this, hir);
127      backwards.add(hir);
128      return;
129    }
130
131    // main case, add to split calculator
132    // ignore replicas other than primary for these checks
133    if (hir.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
134      sc.add(hir);
135    }
136  }
137
138  public void addServer(ServerName server) {
139    this.deployedOn.add(server);
140  }
141
142  public TableName getName() {
143    return tableName;
144  }
145
146  public int getNumRegions() {
147    return sc.getStarts().size() + backwards.size();
148  }
149
150  public synchronized ImmutableList<RegionInfo>
151    getRegionsFromMeta(TreeMap<String, HbckRegionInfo> regionInfoMap) {
152    // lazy loaded, synchronized to ensure a single load
153    if (regionsFromMeta == null) {
154      List<RegionInfo> regions = new ArrayList<>();
155      for (HbckRegionInfo h : regionInfoMap.values()) {
156        if (tableName.equals(h.getTableName())) {
157          if (h.getMetaEntry() != null) {
158            regions.add(h.getMetaEntry().getRegionInfo());
159          }
160        }
161      }
162      regionsFromMeta = Ordering.from(RegionInfo.COMPARATOR).immutableSortedCopy(regions);
163    }
164
165    return regionsFromMeta;
166  }
167
168  class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
169    HbckErrorReporter errors;
170
171    IntegrityFixSuggester(HbckTableInfo ti, HbckErrorReporter errors) {
172      this.errors = errors;
173      setTableInfo(ti);
174    }
175
176    @Override
177    public void handleRegionStartKeyNotEmpty(HbckRegionInfo hi) throws IOException {
178      errors.reportError(HbckErrorReporter.ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
179        "First region should start with an empty key.  You need to "
180          + " create a new region and regioninfo in HDFS to plug the hole.",
181        getTableInfo(), hi);
182    }
183
184    @Override
185    public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
186      errors.reportError(HbckErrorReporter.ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
187        "Last region should end with an empty key. You need to "
188          + "create a new region and regioninfo in HDFS to plug the hole.",
189        getTableInfo());
190    }
191
192    @Override
193    public void handleDegenerateRegion(HbckRegionInfo hi) throws IOException {
194      errors.reportError(HbckErrorReporter.ERROR_CODE.DEGENERATE_REGION,
195        "Region has the same start and end key.", getTableInfo(), hi);
196    }
197
198    @Override
199    public void handleDuplicateStartKeys(HbckRegionInfo r1, HbckRegionInfo r2) throws IOException {
200      byte[] key = r1.getStartKey();
201      // dup start key
202      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
203        "Multiple regions have the same startkey: " + Bytes.toStringBinary(key), getTableInfo(),
204        r1);
205      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
206        "Multiple regions have the same startkey: " + Bytes.toStringBinary(key), getTableInfo(),
207        r2);
208    }
209
210    @Override
211    public void handleSplit(HbckRegionInfo r1, HbckRegionInfo r2) throws IOException {
212      byte[] key = r1.getStartKey();
213      // dup start key
214      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_ENDKEYS,
215        "Multiple regions have the same regionID: " + Bytes.toStringBinary(key), getTableInfo(),
216        r1);
217      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_ENDKEYS,
218        "Multiple regions have the same regionID: " + Bytes.toStringBinary(key), getTableInfo(),
219        r2);
220    }
221
222    @Override
223    public void handleOverlapInRegionChain(HbckRegionInfo hi1, HbckRegionInfo hi2)
224      throws IOException {
225      errors.reportError(HbckErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
226        "There is an overlap in the region chain.", getTableInfo(), hi1, hi2);
227    }
228
229    @Override
230    public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException {
231      errors.reportError(HbckErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN,
232        "There is a hole in the region chain between " + Bytes.toStringBinary(holeStart) + " and "
233          + Bytes.toStringBinary(holeStop) + ".  You need to create a new .regioninfo and region "
234          + "dir in hdfs to plug the hole.");
235    }
236  }
237
238  /**
239   * This handler fixes integrity errors from hdfs information. There are basically three classes of
240   * integrity problems 1) holes, 2) overlaps, and 3) invalid regions. This class overrides methods
241   * that fix holes and the overlap group case. Individual cases of particular overlaps are handled
242   * by the general overlap group merge repair case. If hbase is online, this forces regions offline
243   * before doing merge operations.
244   */
245  class HDFSIntegrityFixer extends IntegrityFixSuggester {
246    Configuration conf;
247
248    boolean fixOverlaps = true;
249
250    HDFSIntegrityFixer(HbckTableInfo ti, HbckErrorReporter errors, Configuration conf,
251      boolean fixHoles, boolean fixOverlaps) {
252      super(ti, errors);
253      this.conf = conf;
254      this.fixOverlaps = fixOverlaps;
255      // TODO properly use fixHoles
256    }
257
258    /**
259     * This is a special case hole -- when the first region of a table is missing from META, HBase
260     * doesn't acknowledge the existance of the table.
261     */
262    @Override
263    public void handleRegionStartKeyNotEmpty(HbckRegionInfo next) throws IOException {
264      errors.reportError(HbckErrorReporter.ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
265        "First region should start with an empty key.  Creating a new "
266          + "region and regioninfo in HDFS to plug the hole.",
267        getTableInfo(), next);
268      TableDescriptor htd = getTableInfo().getTableDescriptor();
269      // from special EMPTY_START_ROW to next region's startKey
270      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName())
271        .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(next.getStartKey()).build();
272
273      // TODO test
274      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
275      LOG.info("Table region start key was not empty.  Created new empty region: " + newRegion + " "
276        + region);
277      hbck.fixes++;
278    }
279
280    @Override
281    public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
282      errors.reportError(HbckErrorReporter.ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
283        "Last region should end with an empty key.  Creating a new "
284          + "region and regioninfo in HDFS to plug the hole.",
285        getTableInfo());
286      TableDescriptor htd = getTableInfo().getTableDescriptor();
287      // from curEndKey to EMPTY_START_ROW
288      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(curEndKey)
289        .setEndKey(HConstants.EMPTY_START_ROW).build();
290
291      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
292      LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion + " "
293        + region);
294      hbck.fixes++;
295    }
296
297    /**
298     * There is a hole in the hdfs regions that violates the table integrity rules. Create a new
299     * empty region that patches the hole.
300     */
301    @Override
302    public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey)
303      throws IOException {
304      errors.reportError(HbckErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN,
305        "There is a hole in the region chain between " + Bytes.toStringBinary(holeStartKey)
306          + " and " + Bytes.toStringBinary(holeStopKey) + ".  Creating a new regioninfo and region "
307          + "dir in hdfs to plug the hole.");
308      TableDescriptor htd = getTableInfo().getTableDescriptor();
309      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName())
310        .setStartKey(holeStartKey).setEndKey(holeStopKey).build();
311      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
312      LOG.info("Plugged hole by creating new empty region: " + newRegion + " " + region);
313      hbck.fixes++;
314    }
315
316    /**
317     * This takes set of overlapping regions and merges them into a single region. This covers cases
318     * like degenerate regions, shared start key, general overlaps, duplicate ranges, and partial
319     * overlapping regions. Cases: - Clean regions that overlap - Only .oldlogs regions (can't find
320     * start/stop range, or figure out) This is basically threadsafe, except for the fixer increment
321     * in mergeOverlaps.
322     */
323    @Override
324    public void handleOverlapGroup(Collection<HbckRegionInfo> overlap) throws IOException {
325      Preconditions.checkNotNull(overlap);
326      Preconditions.checkArgument(overlap.size() > 0);
327
328      if (!this.fixOverlaps) {
329        LOG.warn("Not attempting to repair overlaps.");
330        return;
331      }
332
333      if (overlap.size() > hbck.getMaxMerge()) {
334        LOG.warn(
335          "Overlap group has " + overlap.size() + " overlapping " + "regions which is greater than "
336            + hbck.getMaxMerge() + ", the max number of regions to merge");
337        if (hbck.shouldSidelineBigOverlaps()) {
338          // we only sideline big overlapped groups that exceeds the max number of regions to merge
339          sidelineBigOverlaps(overlap);
340        }
341        return;
342      }
343      if (hbck.shouldRemoveParents()) {
344        removeParentsAndFixSplits(overlap);
345      }
346      mergeOverlaps(overlap);
347    }
348
349    void removeParentsAndFixSplits(Collection<HbckRegionInfo> overlap) throws IOException {
350      Pair<byte[], byte[]> range = null;
351      HbckRegionInfo parent = null;
352      HbckRegionInfo daughterA = null;
353      HbckRegionInfo daughterB = null;
354      Collection<HbckRegionInfo> daughters = new ArrayList<HbckRegionInfo>(overlap);
355
356      String thread = Thread.currentThread().getName();
357      LOG.info("== [" + thread + "] Attempting fix splits in overlap state.");
358
359      // we only can handle a single split per group at the time
360      if (overlap.size() > 3) {
361        LOG.info("Too many overlaps were found on this group, falling back to regular merge.");
362        return;
363      }
364
365      for (HbckRegionInfo hi : overlap) {
366        if (range == null) {
367          range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
368        } else {
369          if (
370            RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getStartKey(), range.getFirst()) < 0
371          ) {
372            range.setFirst(hi.getStartKey());
373          }
374          if (
375            RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getEndKey(), range.getSecond()) > 0
376          ) {
377            range.setSecond(hi.getEndKey());
378          }
379        }
380      }
381
382      LOG.info("This group range is [" + Bytes.toStringBinary(range.getFirst()) + ", "
383        + Bytes.toStringBinary(range.getSecond()) + "]");
384
385      // attempt to find a possible parent for the edge case of a split
386      for (HbckRegionInfo hi : overlap) {
387        if (
388          Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0
389            && Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0
390        ) {
391          LOG.info("This is a parent for this group: " + hi.toString());
392          parent = hi;
393        }
394      }
395
396      // Remove parent regions from daughters collection
397      if (parent != null) {
398        daughters.remove(parent);
399      }
400
401      // Lets verify that daughters share the regionID at split time and they
402      // were created after the parent
403      for (HbckRegionInfo hi : daughters) {
404        if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0) {
405          if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
406            daughterA = hi;
407          }
408        }
409        if (Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
410          if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
411            daughterB = hi;
412          }
413        }
414      }
415
416      // daughters must share the same regionID and we should have a parent too
417      if (
418        daughterA.getHdfsHRI().getRegionId() != daughterB.getHdfsHRI().getRegionId()
419          || parent == null
420      ) {
421        return;
422      }
423
424      FileSystem fs = FileSystem.get(conf);
425      LOG.info("Found parent: " + parent.getRegionNameAsString());
426      LOG.info("Found potential daughter a: " + daughterA.getRegionNameAsString());
427      LOG.info("Found potential daughter b: " + daughterB.getRegionNameAsString());
428      LOG.info("Trying to fix parent in overlap by removing the parent.");
429      try {
430        hbck.closeRegion(parent);
431      } catch (IOException ioe) {
432        LOG.warn("Parent region could not be closed, continuing with regular merge...", ioe);
433        return;
434      } catch (InterruptedException ie) {
435        LOG.warn("Parent region could not be closed, continuing with regular merge...", ie);
436        return;
437      }
438
439      try {
440        hbck.offline(parent.getRegionName());
441      } catch (IOException ioe) {
442        LOG.warn("Unable to offline parent region: " + parent.getRegionNameAsString()
443          + ".  Just continuing with regular merge... ", ioe);
444        return;
445      }
446
447      try {
448        HBaseFsckRepair.removeParentInMeta(conf, parent.getHdfsHRI());
449      } catch (IOException ioe) {
450        LOG.warn("Unable to remove parent region in META: " + parent.getRegionNameAsString()
451          + ".  Just continuing with regular merge... ", ioe);
452        return;
453      }
454
455      hbck.sidelineRegionDir(fs, parent);
456      LOG.info("[" + thread + "] Sidelined parent region dir " + parent.getHdfsRegionDir()
457        + " into " + hbck.getSidelineDir());
458      hbck.debugLsr(parent.getHdfsRegionDir());
459
460      // Make sure we don't have the parents and daughters around
461      overlap.remove(parent);
462      overlap.remove(daughterA);
463      overlap.remove(daughterB);
464
465      LOG.info("Done fixing split.");
466
467    }
468
469    void mergeOverlaps(Collection<HbckRegionInfo> overlap) throws IOException {
470      String thread = Thread.currentThread().getName();
471      LOG.info(
472        "== [" + thread + "] Merging regions into one region: " + Joiner.on(",").join(overlap));
473      // get the min / max range and close all concerned regions
474      Pair<byte[], byte[]> range = null;
475      for (HbckRegionInfo hi : overlap) {
476        if (range == null) {
477          range = new Pair<>(hi.getStartKey(), hi.getEndKey());
478        } else {
479          if (
480            RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getStartKey(), range.getFirst()) < 0
481          ) {
482            range.setFirst(hi.getStartKey());
483          }
484          if (
485            RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getEndKey(), range.getSecond()) > 0
486          ) {
487            range.setSecond(hi.getEndKey());
488          }
489        }
490        // need to close files so delete can happen.
491        LOG.debug("[" + thread + "] Closing region before moving data around: " + hi);
492        LOG.debug("[" + thread + "] Contained region dir before close");
493        hbck.debugLsr(hi.getHdfsRegionDir());
494        try {
495          LOG.info("[" + thread + "] Closing region: " + hi);
496          hbck.closeRegion(hi);
497        } catch (IOException ioe) {
498          LOG.warn("[" + thread + "] Was unable to close region " + hi + ".  Just continuing... ",
499            ioe);
500        } catch (InterruptedException e) {
501          LOG.warn("[" + thread + "] Was unable to close region " + hi + ".  Just continuing... ",
502            e);
503        }
504
505        try {
506          LOG.info("[" + thread + "] Offlining region: " + hi);
507          hbck.offline(hi.getRegionName());
508        } catch (IOException ioe) {
509          LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
510            + ".  Just continuing... ", ioe);
511        }
512      }
513
514      // create new empty container region.
515      TableDescriptor htd = getTableInfo().getTableDescriptor();
516      // from start key to end Key
517      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName())
518        .setStartKey(range.getFirst()).setEndKey(range.getSecond()).build();
519      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
520      LOG.info("[" + thread + "] Created new empty container region: " + newRegion
521        + " to contain regions: " + Joiner.on(",").join(overlap));
522      hbck.debugLsr(region.getRegionFileSystem().getRegionDir());
523
524      // all target regions are closed, should be able to safely cleanup.
525      boolean didFix = false;
526      Path target = region.getRegionFileSystem().getRegionDir();
527      for (HbckRegionInfo contained : overlap) {
528        LOG.info("[" + thread + "] Merging " + contained + " into " + target);
529        int merges = hbck.mergeRegionDirs(target, contained);
530        if (merges > 0) {
531          didFix = true;
532        }
533      }
534      if (didFix) {
535        hbck.fixes++;
536      }
537    }
538
539    /**
540     * Sideline some regions in a big overlap group so that it will have fewer regions, and it is
541     * easier to merge them later on.
542     * @param bigOverlap the overlapped group with regions more than maxMerge
543     */
544    void sidelineBigOverlaps(Collection<HbckRegionInfo> bigOverlap) throws IOException {
545      int overlapsToSideline = bigOverlap.size() - hbck.getMaxMerge();
546      if (overlapsToSideline > hbck.getMaxOverlapsToSideline()) {
547        overlapsToSideline = hbck.getMaxOverlapsToSideline();
548      }
549      List<HbckRegionInfo> regionsToSideline =
550        RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
551      FileSystem fs = FileSystem.get(conf);
552      for (HbckRegionInfo regionToSideline : regionsToSideline) {
553        try {
554          LOG.info("Closing region: " + regionToSideline);
555          hbck.closeRegion(regionToSideline);
556        } catch (IOException ioe) {
557          LOG.warn("Was unable to close region " + regionToSideline + ".  Just continuing... ",
558            ioe);
559        } catch (InterruptedException e) {
560          LOG.warn("Was unable to close region " + regionToSideline + ".  Just continuing... ", e);
561        }
562
563        try {
564          LOG.info("Offlining region: " + regionToSideline);
565          hbck.offline(regionToSideline.getRegionName());
566        } catch (IOException ioe) {
567          LOG.warn(
568            "Unable to offline region from master: " + regionToSideline + ".  Just continuing... ",
569            ioe);
570        }
571
572        LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
573        Path sidelineRegionDir = hbck.sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
574        if (sidelineRegionDir != null) {
575          sidelinedRegions.put(sidelineRegionDir, regionToSideline);
576          LOG.info("After sidelined big overlapped region: "
577            + regionToSideline.getRegionNameAsString() + " to " + sidelineRegionDir.toString());
578          hbck.fixes++;
579        }
580      }
581    }
582  }
583
584  /**
585   * Check the region chain (from META) of this table. We are looking for holes, overlaps, and
586   * cycles.
587   * @return false if there are errors
588   */
589  public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
590    // When table is disabled no need to check for the region chain. Some of the regions
591    // accidently if deployed, this below code might report some issues like missing start
592    // or end regions or region hole in chain and may try to fix which is unwanted.
593    if (hbck.isTableDisabled(this.tableName)) {
594      return true;
595    }
596    int originalErrorsCount = hbck.getErrors().getErrorList().size();
597    Multimap<byte[], HbckRegionInfo> regions = sc.calcCoverage();
598    SortedSet<byte[]> splits = sc.getSplits();
599
600    byte[] prevKey = null;
601    byte[] problemKey = null;
602
603    if (splits.isEmpty()) {
604      // no region for this table
605      handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
606    }
607
608    for (byte[] key : splits) {
609      Collection<HbckRegionInfo> ranges = regions.get(key);
610      if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
611        for (HbckRegionInfo rng : ranges) {
612          handler.handleRegionStartKeyNotEmpty(rng);
613        }
614      }
615
616      // check for degenerate ranges
617      for (HbckRegionInfo rng : ranges) {
618        // special endkey case converts '' to null
619        byte[] endKey = rng.getEndKey();
620        endKey = (endKey.length == 0) ? null : endKey;
621        if (Bytes.equals(rng.getStartKey(), endKey)) {
622          handler.handleDegenerateRegion(rng);
623        }
624      }
625
626      if (ranges.size() == 1) {
627        // this split key is ok -- no overlap, not a hole.
628        if (problemKey != null) {
629          LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
630        }
631        problemKey = null; // fell through, no more problem.
632      } else if (ranges.size() > 1) {
633        // set the new problem key group name, if already have problem key, just
634        // keep using it.
635        if (problemKey == null) {
636          // only for overlap regions.
637          LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
638          problemKey = key;
639        }
640        overlapGroups.putAll(problemKey, ranges);
641
642        // record errors
643        ArrayList<HbckRegionInfo> subRange = new ArrayList<>(ranges);
644        // this dumb and n^2 but this shouldn't happen often
645        for (HbckRegionInfo r1 : ranges) {
646          if (r1.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
647            continue;
648          }
649          subRange.remove(r1);
650          for (HbckRegionInfo r2 : subRange) {
651            if (r2.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
652              continue;
653            }
654            // general case of same start key
655            if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey()) == 0) {
656              handler.handleDuplicateStartKeys(r1, r2);
657            } else if (
658              Bytes.compareTo(r1.getEndKey(), r2.getStartKey()) == 0
659                && r1.getHdfsHRI().getRegionId() == r2.getHdfsHRI().getRegionId()
660            ) {
661              LOG.info("this is a split, log to splits");
662              handler.handleSplit(r1, r2);
663            } else {
664              // overlap
665              handler.handleOverlapInRegionChain(r1, r2);
666            }
667          }
668        }
669
670      } else if (ranges.isEmpty()) {
671        if (problemKey != null) {
672          LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
673        }
674        problemKey = null;
675
676        byte[] holeStopKey = sc.getSplits().higher(key);
677        // if higher key is null we reached the top.
678        if (holeStopKey != null) {
679          // hole
680          handler.handleHoleInRegionChain(key, holeStopKey);
681        }
682      }
683      prevKey = key;
684    }
685
686    // When the last region of a table is proper and having an empty end key, 'prevKey'
687    // will be null.
688    if (prevKey != null) {
689      handler.handleRegionEndKeyNotEmpty(prevKey);
690    }
691
692    // TODO fold this into the TableIntegrityHandler
693    if (hbck.getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
694      boolean ok = handleOverlapsParallel(handler, prevKey);
695      if (!ok) {
696        return false;
697      }
698    } else {
699      for (Collection<HbckRegionInfo> overlap : overlapGroups.asMap().values()) {
700        handler.handleOverlapGroup(overlap);
701      }
702    }
703
704    if (HBaseFsck.shouldDisplayFullReport()) {
705      // do full region split map dump
706      hbck.getErrors().print("---- Table '" + this.tableName + "': region split map");
707      dump(splits, regions);
708      hbck.getErrors().print("---- Table '" + this.tableName + "': overlap groups");
709      dumpOverlapProblems(overlapGroups);
710      hbck.getErrors().print("There are " + overlapGroups.keySet().size() + " overlap groups with "
711        + overlapGroups.size() + " overlapping regions");
712    }
713    if (!sidelinedRegions.isEmpty()) {
714      LOG.warn("Sidelined big overlapped regions, please bulk load them!");
715      hbck.getErrors()
716        .print("---- Table '" + this.tableName + "': sidelined big overlapped regions");
717      dumpSidelinedRegions(sidelinedRegions);
718    }
719    return hbck.getErrors().getErrorList().size() == originalErrorsCount;
720  }
721
722  private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
723    throws IOException {
724    // we parallelize overlap handler for the case we have lots of groups to fix. We can
725    // safely assume each group is independent.
726    List<HBaseFsck.WorkItemOverlapMerge> merges = new ArrayList<>(overlapGroups.size());
727    List<Future<Void>> rets;
728    for (Collection<HbckRegionInfo> overlap : overlapGroups.asMap().values()) {
729      //
730      merges.add(new HBaseFsck.WorkItemOverlapMerge(overlap, handler));
731    }
732    try {
733      rets = hbck.executor.invokeAll(merges);
734    } catch (InterruptedException e) {
735      LOG.error("Overlap merges were interrupted", e);
736      return false;
737    }
738    for (int i = 0; i < merges.size(); i++) {
739      HBaseFsck.WorkItemOverlapMerge work = merges.get(i);
740      Future<Void> f = rets.get(i);
741      try {
742        f.get();
743      } catch (ExecutionException e) {
744        LOG.warn("Failed to merge overlap group" + work, e.getCause());
745      } catch (InterruptedException e) {
746        LOG.error("Waiting for overlap merges was interrupted", e);
747        return false;
748      }
749    }
750    return true;
751  }
752
753  /**
754   * This dumps data in a visually reasonable way for visual debugging
755   */
756  private void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckRegionInfo> regions) {
757    // we display this way because the last end key should be displayed as well.
758    StringBuilder sb = new StringBuilder();
759    for (byte[] k : splits) {
760      sb.setLength(0); // clear out existing buffer, if any.
761      sb.append(Bytes.toStringBinary(k) + ":\t");
762      for (HbckRegionInfo r : regions.get(k)) {
763        sb.append("[ " + r.toString() + ", " + Bytes.toStringBinary(r.getEndKey()) + "]\t");
764      }
765      hbck.getErrors().print(sb.toString());
766    }
767  }
768
769  private void dumpOverlapProblems(Multimap<byte[], HbckRegionInfo> regions) {
770    // we display this way because the last end key should be displayed as
771    // well.
772    for (byte[] k : regions.keySet()) {
773      hbck.getErrors().print(Bytes.toStringBinary(k) + ":");
774      for (HbckRegionInfo r : regions.get(k)) {
775        hbck.getErrors()
776          .print("[ " + r.toString() + ", " + Bytes.toStringBinary(r.getEndKey()) + "]");
777      }
778      hbck.getErrors().print("----");
779    }
780  }
781
782  private void dumpSidelinedRegions(Map<Path, HbckRegionInfo> regions) {
783    for (Map.Entry<Path, HbckRegionInfo> entry : regions.entrySet()) {
784      TableName tableName = entry.getValue().getTableName();
785      Path path = entry.getKey();
786      hbck.getErrors().print("This sidelined region dir should be bulk loaded: " + path.toString());
787      hbck.getErrors().print("Bulk load command looks like: " + BulkLoadHFilesTool.NAME + " "
788        + path.toUri().getPath() + " " + tableName);
789    }
790  }
791}