001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Collection;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.List;
026import java.util.Map;
027import java.util.Set;
028import java.util.SortedSet;
029import java.util.TreeMap;
030import java.util.TreeSet;
031import java.util.concurrent.ExecutionException;
032import java.util.concurrent.Future;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.ServerName;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.RegionInfoBuilder;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.regionserver.HRegion;
043import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool;
044import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
045import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
046import org.apache.yetus.audience.InterfaceAudience;
047import org.apache.yetus.audience.InterfaceStability;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
052import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
053import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
054import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
055import org.apache.hbase.thirdparty.com.google.common.collect.Ordering;
056import org.apache.hbase.thirdparty.com.google.common.collect.TreeMultimap;
057
058/**
059 * Maintain information about a particular table.
060 */
061@InterfaceAudience.Private
062@InterfaceStability.Evolving
063public class HbckTableInfo {
064  private static final Logger LOG = LoggerFactory.getLogger(HbckTableInfo.class.getName());
065
066  private static final String TO_BE_LOADED = "to_be_loaded";
067
068  TableName tableName;
069  TreeSet<ServerName> deployedOn;
070
071  // backwards regions
072  final List<HbckRegionInfo> backwards = new ArrayList<>();
073
074  // sidelined big overlapped regions
075  final Map<Path, HbckRegionInfo> sidelinedRegions = new HashMap<>();
076
077  // region split calculator
078  final RegionSplitCalculator<HbckRegionInfo> sc =
079    new RegionSplitCalculator<>(HbckRegionInfo.COMPARATOR);
080
081  // Histogram of different TableDescriptors found. Ideally there is only one!
082  final Set<TableDescriptor> htds = new HashSet<>();
083
084  // key = start split, values = set of splits in problem group
085  final Multimap<byte[], HbckRegionInfo> overlapGroups =
086    TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, HbckRegionInfo.COMPARATOR);
087
088  // list of regions derived from meta entries.
089  private ImmutableList<RegionInfo> regionsFromMeta = null;
090
091  HBaseFsck hbck;
092
093  HbckTableInfo(TableName name, HBaseFsck hbck) {
094    this.tableName = name;
095    this.hbck = hbck;
096    deployedOn = new TreeSet<>();
097  }
098
099  /**
100   * @return descriptor common to all regions. null if are none or multiple!
101   */
102  TableDescriptor getTableDescriptor() {
103    if (htds.size() == 1) {
104      return (TableDescriptor) htds.toArray()[0];
105    } else {
106      LOG.error(
107        "None/Multiple table descriptors found for table '" + tableName + "' regions: " + htds);
108    }
109    return null;
110  }
111
112  public void addRegionInfo(HbckRegionInfo hir) {
113    if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
114      // end key is absolute end key, just add it.
115      // ignore replicas other than primary for these checks
116      if (hir.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
117        sc.add(hir);
118      }
119      return;
120    }
121
122    // if not the absolute end key, check for cycle
123    if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
124      hbck.getErrors().reportError(HbckErrorReporter.ERROR_CODE.REGION_CYCLE,
125        String.format(
126          "The endkey for this region comes before the " + "startkey, startkey=%s, endkey=%s",
127          Bytes.toStringBinary(hir.getStartKey()), Bytes.toStringBinary(hir.getEndKey())),
128        this, hir);
129      backwards.add(hir);
130      return;
131    }
132
133    // main case, add to split calculator
134    // ignore replicas other than primary for these checks
135    if (hir.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
136      sc.add(hir);
137    }
138  }
139
140  public void addServer(ServerName server) {
141    this.deployedOn.add(server);
142  }
143
144  public TableName getName() {
145    return tableName;
146  }
147
148  public int getNumRegions() {
149    return sc.getStarts().size() + backwards.size();
150  }
151
152  public synchronized ImmutableList<RegionInfo>
153    getRegionsFromMeta(TreeMap<String, HbckRegionInfo> regionInfoMap) {
154    // lazy loaded, synchronized to ensure a single load
155    if (regionsFromMeta == null) {
156      List<RegionInfo> regions = new ArrayList<>();
157      for (HbckRegionInfo h : regionInfoMap.values()) {
158        if (tableName.equals(h.getTableName())) {
159          if (h.getMetaEntry() != null) {
160            regions.add(h.getMetaEntry().getRegionInfo());
161          }
162        }
163      }
164      regionsFromMeta = Ordering.from(RegionInfo.COMPARATOR).immutableSortedCopy(regions);
165    }
166
167    return regionsFromMeta;
168  }
169
170  class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
171    HbckErrorReporter errors;
172
173    IntegrityFixSuggester(HbckTableInfo ti, HbckErrorReporter errors) {
174      this.errors = errors;
175      setTableInfo(ti);
176    }
177
178    @Override
179    public void handleRegionStartKeyNotEmpty(HbckRegionInfo hi) throws IOException {
180      errors.reportError(HbckErrorReporter.ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
181        "First region should start with an empty key.  You need to "
182          + " create a new region and regioninfo in HDFS to plug the hole.",
183        getTableInfo(), hi);
184    }
185
186    @Override
187    public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
188      errors.reportError(HbckErrorReporter.ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
189        "Last region should end with an empty key. You need to "
190          + "create a new region and regioninfo in HDFS to plug the hole.",
191        getTableInfo());
192    }
193
194    @Override
195    public void handleDegenerateRegion(HbckRegionInfo hi) throws IOException {
196      errors.reportError(HbckErrorReporter.ERROR_CODE.DEGENERATE_REGION,
197        "Region has the same start and end key.", getTableInfo(), hi);
198    }
199
200    @Override
201    public void handleDuplicateStartKeys(HbckRegionInfo r1, HbckRegionInfo r2) throws IOException {
202      byte[] key = r1.getStartKey();
203      // dup start key
204      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
205        "Multiple regions have the same startkey: " + Bytes.toStringBinary(key), getTableInfo(),
206        r1);
207      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
208        "Multiple regions have the same startkey: " + Bytes.toStringBinary(key), getTableInfo(),
209        r2);
210    }
211
212    @Override
213    public void handleSplit(HbckRegionInfo r1, HbckRegionInfo r2) throws IOException {
214      byte[] key = r1.getStartKey();
215      // dup start key
216      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_ENDKEYS,
217        "Multiple regions have the same regionID: " + Bytes.toStringBinary(key), getTableInfo(),
218        r1);
219      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_ENDKEYS,
220        "Multiple regions have the same regionID: " + Bytes.toStringBinary(key), getTableInfo(),
221        r2);
222    }
223
224    @Override
225    public void handleOverlapInRegionChain(HbckRegionInfo hi1, HbckRegionInfo hi2)
226      throws IOException {
227      errors.reportError(HbckErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
228        "There is an overlap in the region chain.", getTableInfo(), hi1, hi2);
229    }
230
231    @Override
232    public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException {
233      errors.reportError(HbckErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN,
234        "There is a hole in the region chain between " + Bytes.toStringBinary(holeStart) + " and "
235          + Bytes.toStringBinary(holeStop) + ".  You need to create a new .regioninfo and region "
236          + "dir in hdfs to plug the hole.");
237    }
238  }
239
240  /**
241   * This handler fixes integrity errors from hdfs information. There are basically three classes of
242   * integrity problems 1) holes, 2) overlaps, and 3) invalid regions. This class overrides methods
243   * that fix holes and the overlap group case. Individual cases of particular overlaps are handled
244   * by the general overlap group merge repair case. If hbase is online, this forces regions offline
245   * before doing merge operations.
246   */
247  class HDFSIntegrityFixer extends IntegrityFixSuggester {
248    Configuration conf;
249
250    boolean fixOverlaps = true;
251
252    HDFSIntegrityFixer(HbckTableInfo ti, HbckErrorReporter errors, Configuration conf,
253      boolean fixHoles, boolean fixOverlaps) {
254      super(ti, errors);
255      this.conf = conf;
256      this.fixOverlaps = fixOverlaps;
257      // TODO properly use fixHoles
258    }
259
260    /**
261     * This is a special case hole -- when the first region of a table is missing from META, HBase
262     * doesn't acknowledge the existance of the table.
263     */
264    @Override
265    public void handleRegionStartKeyNotEmpty(HbckRegionInfo next) throws IOException {
266      errors.reportError(HbckErrorReporter.ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
267        "First region should start with an empty key.  Creating a new "
268          + "region and regioninfo in HDFS to plug the hole.",
269        getTableInfo(), next);
270      TableDescriptor htd = getTableInfo().getTableDescriptor();
271      // from special EMPTY_START_ROW to next region's startKey
272      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName())
273        .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(next.getStartKey()).build();
274
275      // TODO test
276      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
277      LOG.info("Table region start key was not empty.  Created new empty region: " + newRegion + " "
278        + region);
279      hbck.fixes++;
280    }
281
282    @Override
283    public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
284      errors.reportError(HbckErrorReporter.ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
285        "Last region should end with an empty key.  Creating a new "
286          + "region and regioninfo in HDFS to plug the hole.",
287        getTableInfo());
288      TableDescriptor htd = getTableInfo().getTableDescriptor();
289      // from curEndKey to EMPTY_START_ROW
290      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(curEndKey)
291        .setEndKey(HConstants.EMPTY_START_ROW).build();
292
293      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
294      LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion + " "
295        + region);
296      hbck.fixes++;
297    }
298
299    /**
300     * There is a hole in the hdfs regions that violates the table integrity rules. Create a new
301     * empty region that patches the hole.
302     */
303    @Override
304    public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey)
305      throws IOException {
306      errors.reportError(HbckErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN,
307        "There is a hole in the region chain between " + Bytes.toStringBinary(holeStartKey)
308          + " and " + Bytes.toStringBinary(holeStopKey) + ".  Creating a new regioninfo and region "
309          + "dir in hdfs to plug the hole.");
310      TableDescriptor htd = getTableInfo().getTableDescriptor();
311      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName())
312        .setStartKey(holeStartKey).setEndKey(holeStopKey).build();
313      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
314      LOG.info("Plugged hole by creating new empty region: " + newRegion + " " + region);
315      hbck.fixes++;
316    }
317
318    /**
319     * This takes set of overlapping regions and merges them into a single region. This covers cases
320     * like degenerate regions, shared start key, general overlaps, duplicate ranges, and partial
321     * overlapping regions. Cases: - Clean regions that overlap - Only .oldlogs regions (can't find
322     * start/stop range, or figure out) This is basically threadsafe, except for the fixer increment
323     * in mergeOverlaps.
324     */
325    @Override
326    public void handleOverlapGroup(Collection<HbckRegionInfo> overlap) throws IOException {
327      Preconditions.checkNotNull(overlap);
328      Preconditions.checkArgument(overlap.size() > 0);
329
330      if (!this.fixOverlaps) {
331        LOG.warn("Not attempting to repair overlaps.");
332        return;
333      }
334
335      if (overlap.size() > hbck.getMaxMerge()) {
336        LOG.warn(
337          "Overlap group has " + overlap.size() + " overlapping " + "regions which is greater than "
338            + hbck.getMaxMerge() + ", the max number of regions to merge");
339        if (hbck.shouldSidelineBigOverlaps()) {
340          // we only sideline big overlapped groups that exceeds the max number of regions to merge
341          sidelineBigOverlaps(overlap);
342        }
343        return;
344      }
345      if (hbck.shouldRemoveParents()) {
346        removeParentsAndFixSplits(overlap);
347      }
348      mergeOverlaps(overlap);
349    }
350
351    void removeParentsAndFixSplits(Collection<HbckRegionInfo> overlap) throws IOException {
352      Pair<byte[], byte[]> range = null;
353      HbckRegionInfo parent = null;
354      HbckRegionInfo daughterA = null;
355      HbckRegionInfo daughterB = null;
356      Collection<HbckRegionInfo> daughters = new ArrayList<HbckRegionInfo>(overlap);
357
358      String thread = Thread.currentThread().getName();
359      LOG.info("== [" + thread + "] Attempting fix splits in overlap state.");
360
361      // we only can handle a single split per group at the time
362      if (overlap.size() > 3) {
363        LOG.info("Too many overlaps were found on this group, falling back to regular merge.");
364        return;
365      }
366
367      for (HbckRegionInfo hi : overlap) {
368        if (range == null) {
369          range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
370        } else {
371          if (
372            RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getStartKey(), range.getFirst()) < 0
373          ) {
374            range.setFirst(hi.getStartKey());
375          }
376          if (
377            RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getEndKey(), range.getSecond()) > 0
378          ) {
379            range.setSecond(hi.getEndKey());
380          }
381        }
382      }
383
384      LOG.info("This group range is [" + Bytes.toStringBinary(range.getFirst()) + ", "
385        + Bytes.toStringBinary(range.getSecond()) + "]");
386
387      // attempt to find a possible parent for the edge case of a split
388      for (HbckRegionInfo hi : overlap) {
389        if (
390          Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0
391            && Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0
392        ) {
393          LOG.info("This is a parent for this group: " + hi.toString());
394          parent = hi;
395        }
396      }
397
398      // Remove parent regions from daughters collection
399      if (parent != null) {
400        daughters.remove(parent);
401      }
402
403      // Lets verify that daughters share the regionID at split time and they
404      // were created after the parent
405      for (HbckRegionInfo hi : daughters) {
406        if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0) {
407          if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
408            daughterA = hi;
409          }
410        }
411        if (Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
412          if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
413            daughterB = hi;
414          }
415        }
416      }
417
418      // daughters must share the same regionID and we should have a parent too
419      if (
420        daughterA.getHdfsHRI().getRegionId() != daughterB.getHdfsHRI().getRegionId()
421          || parent == null
422      ) {
423        return;
424      }
425
426      FileSystem fs = FileSystem.get(conf);
427      LOG.info("Found parent: " + parent.getRegionNameAsString());
428      LOG.info("Found potential daughter a: " + daughterA.getRegionNameAsString());
429      LOG.info("Found potential daughter b: " + daughterB.getRegionNameAsString());
430      LOG.info("Trying to fix parent in overlap by removing the parent.");
431      try {
432        hbck.closeRegion(parent);
433      } catch (IOException ioe) {
434        LOG.warn("Parent region could not be closed, continuing with regular merge...", ioe);
435        return;
436      } catch (InterruptedException ie) {
437        LOG.warn("Parent region could not be closed, continuing with regular merge...", ie);
438        return;
439      }
440
441      try {
442        hbck.offline(parent.getRegionName());
443      } catch (IOException ioe) {
444        LOG.warn("Unable to offline parent region: " + parent.getRegionNameAsString()
445          + ".  Just continuing with regular merge... ", ioe);
446        return;
447      }
448
449      try {
450        HBaseFsckRepair.removeParentInMeta(conf, parent.getHdfsHRI());
451      } catch (IOException ioe) {
452        LOG.warn("Unable to remove parent region in META: " + parent.getRegionNameAsString()
453          + ".  Just continuing with regular merge... ", ioe);
454        return;
455      }
456
457      hbck.sidelineRegionDir(fs, parent);
458      LOG.info("[" + thread + "] Sidelined parent region dir " + parent.getHdfsRegionDir()
459        + " into " + hbck.getSidelineDir());
460      hbck.debugLsr(parent.getHdfsRegionDir());
461
462      // Make sure we don't have the parents and daughters around
463      overlap.remove(parent);
464      overlap.remove(daughterA);
465      overlap.remove(daughterB);
466
467      LOG.info("Done fixing split.");
468
469    }
470
471    void mergeOverlaps(Collection<HbckRegionInfo> overlap) throws IOException {
472      String thread = Thread.currentThread().getName();
473      LOG.info(
474        "== [" + thread + "] Merging regions into one region: " + Joiner.on(",").join(overlap));
475      // get the min / max range and close all concerned regions
476      Pair<byte[], byte[]> range = null;
477      for (HbckRegionInfo hi : overlap) {
478        if (range == null) {
479          range = new Pair<>(hi.getStartKey(), hi.getEndKey());
480        } else {
481          if (
482            RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getStartKey(), range.getFirst()) < 0
483          ) {
484            range.setFirst(hi.getStartKey());
485          }
486          if (
487            RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getEndKey(), range.getSecond()) > 0
488          ) {
489            range.setSecond(hi.getEndKey());
490          }
491        }
492        // need to close files so delete can happen.
493        LOG.debug("[" + thread + "] Closing region before moving data around: " + hi);
494        LOG.debug("[" + thread + "] Contained region dir before close");
495        hbck.debugLsr(hi.getHdfsRegionDir());
496        try {
497          LOG.info("[" + thread + "] Closing region: " + hi);
498          hbck.closeRegion(hi);
499        } catch (IOException ioe) {
500          LOG.warn("[" + thread + "] Was unable to close region " + hi + ".  Just continuing... ",
501            ioe);
502        } catch (InterruptedException e) {
503          LOG.warn("[" + thread + "] Was unable to close region " + hi + ".  Just continuing... ",
504            e);
505        }
506
507        try {
508          LOG.info("[" + thread + "] Offlining region: " + hi);
509          hbck.offline(hi.getRegionName());
510        } catch (IOException ioe) {
511          LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
512            + ".  Just continuing... ", ioe);
513        }
514      }
515
516      // create new empty container region.
517      TableDescriptor htd = getTableInfo().getTableDescriptor();
518      // from start key to end Key
519      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName())
520        .setStartKey(range.getFirst()).setEndKey(range.getSecond()).build();
521      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
522      LOG.info("[" + thread + "] Created new empty container region: " + newRegion
523        + " to contain regions: " + Joiner.on(",").join(overlap));
524      hbck.debugLsr(region.getRegionFileSystem().getRegionDir());
525
526      // all target regions are closed, should be able to safely cleanup.
527      boolean didFix = false;
528      Path target = region.getRegionFileSystem().getRegionDir();
529      for (HbckRegionInfo contained : overlap) {
530        LOG.info("[" + thread + "] Merging " + contained + " into " + target);
531        int merges = hbck.mergeRegionDirs(target, contained);
532        if (merges > 0) {
533          didFix = true;
534        }
535      }
536      if (didFix) {
537        hbck.fixes++;
538      }
539    }
540
541    /**
542     * Sideline some regions in a big overlap group so that it will have fewer regions, and it is
543     * easier to merge them later on.
544     * @param bigOverlap the overlapped group with regions more than maxMerge
545     */
546    void sidelineBigOverlaps(Collection<HbckRegionInfo> bigOverlap) throws IOException {
547      int overlapsToSideline = bigOverlap.size() - hbck.getMaxMerge();
548      if (overlapsToSideline > hbck.getMaxOverlapsToSideline()) {
549        overlapsToSideline = hbck.getMaxOverlapsToSideline();
550      }
551      List<HbckRegionInfo> regionsToSideline =
552        RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
553      FileSystem fs = FileSystem.get(conf);
554      for (HbckRegionInfo regionToSideline : regionsToSideline) {
555        try {
556          LOG.info("Closing region: " + regionToSideline);
557          hbck.closeRegion(regionToSideline);
558        } catch (IOException ioe) {
559          LOG.warn("Was unable to close region " + regionToSideline + ".  Just continuing... ",
560            ioe);
561        } catch (InterruptedException e) {
562          LOG.warn("Was unable to close region " + regionToSideline + ".  Just continuing... ", e);
563        }
564
565        try {
566          LOG.info("Offlining region: " + regionToSideline);
567          hbck.offline(regionToSideline.getRegionName());
568        } catch (IOException ioe) {
569          LOG.warn(
570            "Unable to offline region from master: " + regionToSideline + ".  Just continuing... ",
571            ioe);
572        }
573
574        LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
575        Path sidelineRegionDir = hbck.sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
576        if (sidelineRegionDir != null) {
577          sidelinedRegions.put(sidelineRegionDir, regionToSideline);
578          LOG.info("After sidelined big overlapped region: "
579            + regionToSideline.getRegionNameAsString() + " to " + sidelineRegionDir.toString());
580          hbck.fixes++;
581        }
582      }
583    }
584  }
585
586  /**
587   * Check the region chain (from META) of this table. We are looking for holes, overlaps, and
588   * cycles.
589   * @return false if there are errors
590   */
591  public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
592    // When table is disabled no need to check for the region chain. Some of the regions
593    // accidently if deployed, this below code might report some issues like missing start
594    // or end regions or region hole in chain and may try to fix which is unwanted.
595    if (hbck.isTableDisabled(this.tableName)) {
596      return true;
597    }
598    int originalErrorsCount = hbck.getErrors().getErrorList().size();
599    Multimap<byte[], HbckRegionInfo> regions = sc.calcCoverage();
600    SortedSet<byte[]> splits = sc.getSplits();
601
602    byte[] prevKey = null;
603    byte[] problemKey = null;
604
605    if (splits.isEmpty()) {
606      // no region for this table
607      handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
608    }
609
610    for (byte[] key : splits) {
611      Collection<HbckRegionInfo> ranges = regions.get(key);
612      if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
613        for (HbckRegionInfo rng : ranges) {
614          handler.handleRegionStartKeyNotEmpty(rng);
615        }
616      }
617
618      // check for degenerate ranges
619      for (HbckRegionInfo rng : ranges) {
620        // special endkey case converts '' to null
621        byte[] endKey = rng.getEndKey();
622        endKey = (endKey.length == 0) ? null : endKey;
623        if (Bytes.equals(rng.getStartKey(), endKey)) {
624          handler.handleDegenerateRegion(rng);
625        }
626      }
627
628      if (ranges.size() == 1) {
629        // this split key is ok -- no overlap, not a hole.
630        if (problemKey != null) {
631          LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
632        }
633        problemKey = null; // fell through, no more problem.
634      } else if (ranges.size() > 1) {
635        // set the new problem key group name, if already have problem key, just
636        // keep using it.
637        if (problemKey == null) {
638          // only for overlap regions.
639          LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
640          problemKey = key;
641        }
642        overlapGroups.putAll(problemKey, ranges);
643
644        // record errors
645        ArrayList<HbckRegionInfo> subRange = new ArrayList<>(ranges);
646        // this dumb and n^2 but this shouldn't happen often
647        for (HbckRegionInfo r1 : ranges) {
648          if (r1.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
649            continue;
650          }
651          subRange.remove(r1);
652          for (HbckRegionInfo r2 : subRange) {
653            if (r2.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
654              continue;
655            }
656            // general case of same start key
657            if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey()) == 0) {
658              handler.handleDuplicateStartKeys(r1, r2);
659            } else if (
660              Bytes.compareTo(r1.getEndKey(), r2.getStartKey()) == 0
661                && r1.getHdfsHRI().getRegionId() == r2.getHdfsHRI().getRegionId()
662            ) {
663              LOG.info("this is a split, log to splits");
664              handler.handleSplit(r1, r2);
665            } else {
666              // overlap
667              handler.handleOverlapInRegionChain(r1, r2);
668            }
669          }
670        }
671
672      } else if (ranges.isEmpty()) {
673        if (problemKey != null) {
674          LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
675        }
676        problemKey = null;
677
678        byte[] holeStopKey = sc.getSplits().higher(key);
679        // if higher key is null we reached the top.
680        if (holeStopKey != null) {
681          // hole
682          handler.handleHoleInRegionChain(key, holeStopKey);
683        }
684      }
685      prevKey = key;
686    }
687
688    // When the last region of a table is proper and having an empty end key, 'prevKey'
689    // will be null.
690    if (prevKey != null) {
691      handler.handleRegionEndKeyNotEmpty(prevKey);
692    }
693
694    // TODO fold this into the TableIntegrityHandler
695    if (hbck.getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
696      boolean ok = handleOverlapsParallel(handler, prevKey);
697      if (!ok) {
698        return false;
699      }
700    } else {
701      for (Collection<HbckRegionInfo> overlap : overlapGroups.asMap().values()) {
702        handler.handleOverlapGroup(overlap);
703      }
704    }
705
706    if (HBaseFsck.shouldDisplayFullReport()) {
707      // do full region split map dump
708      hbck.getErrors().print("---- Table '" + this.tableName + "': region split map");
709      dump(splits, regions);
710      hbck.getErrors().print("---- Table '" + this.tableName + "': overlap groups");
711      dumpOverlapProblems(overlapGroups);
712      hbck.getErrors().print("There are " + overlapGroups.keySet().size() + " overlap groups with "
713        + overlapGroups.size() + " overlapping regions");
714    }
715    if (!sidelinedRegions.isEmpty()) {
716      LOG.warn("Sidelined big overlapped regions, please bulk load them!");
717      hbck.getErrors()
718        .print("---- Table '" + this.tableName + "': sidelined big overlapped regions");
719      dumpSidelinedRegions(sidelinedRegions);
720    }
721    return hbck.getErrors().getErrorList().size() == originalErrorsCount;
722  }
723
724  private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
725    throws IOException {
726    // we parallelize overlap handler for the case we have lots of groups to fix. We can
727    // safely assume each group is independent.
728    List<HBaseFsck.WorkItemOverlapMerge> merges = new ArrayList<>(overlapGroups.size());
729    List<Future<Void>> rets;
730    for (Collection<HbckRegionInfo> overlap : overlapGroups.asMap().values()) {
731      //
732      merges.add(new HBaseFsck.WorkItemOverlapMerge(overlap, handler));
733    }
734    try {
735      rets = hbck.executor.invokeAll(merges);
736    } catch (InterruptedException e) {
737      LOG.error("Overlap merges were interrupted", e);
738      return false;
739    }
740    for (int i = 0; i < merges.size(); i++) {
741      HBaseFsck.WorkItemOverlapMerge work = merges.get(i);
742      Future<Void> f = rets.get(i);
743      try {
744        f.get();
745      } catch (ExecutionException e) {
746        LOG.warn("Failed to merge overlap group" + work, e.getCause());
747      } catch (InterruptedException e) {
748        LOG.error("Waiting for overlap merges was interrupted", e);
749        return false;
750      }
751    }
752    return true;
753  }
754
755  /**
756   * This dumps data in a visually reasonable way for visual debugging
757   */
758  private void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckRegionInfo> regions) {
759    // we display this way because the last end key should be displayed as well.
760    StringBuilder sb = new StringBuilder();
761    for (byte[] k : splits) {
762      sb.setLength(0); // clear out existing buffer, if any.
763      sb.append(Bytes.toStringBinary(k) + ":\t");
764      for (HbckRegionInfo r : regions.get(k)) {
765        sb.append("[ " + r.toString() + ", " + Bytes.toStringBinary(r.getEndKey()) + "]\t");
766      }
767      hbck.getErrors().print(sb.toString());
768    }
769  }
770
771  private void dumpOverlapProblems(Multimap<byte[], HbckRegionInfo> regions) {
772    // we display this way because the last end key should be displayed as
773    // well.
774    for (byte[] k : regions.keySet()) {
775      hbck.getErrors().print(Bytes.toStringBinary(k) + ":");
776      for (HbckRegionInfo r : regions.get(k)) {
777        hbck.getErrors()
778          .print("[ " + r.toString() + ", " + Bytes.toStringBinary(r.getEndKey()) + "]");
779      }
780      hbck.getErrors().print("----");
781    }
782  }
783
784  private void dumpSidelinedRegions(Map<Path, HbckRegionInfo> regions) {
785    for (Map.Entry<Path, HbckRegionInfo> entry : regions.entrySet()) {
786      TableName tableName = entry.getValue().getTableName();
787      Path path = entry.getKey();
788      hbck.getErrors().print("This sidelined region dir should be bulk loaded: " + path.toString());
789      hbck.getErrors().print("Bulk load command looks like: " + BulkLoadHFilesTool.NAME + " "
790        + path.toUri().getPath() + " " + tableName);
791    }
792  }
793}