001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Collection;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.List;
026import java.util.Map;
027import java.util.Set;
028import java.util.SortedSet;
029import java.util.TreeMap;
030import java.util.TreeSet;
031import java.util.concurrent.ExecutionException;
032import java.util.concurrent.Future;
033
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.fs.FileSystem;
036import org.apache.hadoop.fs.Path;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.ServerName;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.client.RegionInfo;
041import org.apache.hadoop.hbase.client.RegionInfoBuilder;
042import org.apache.hadoop.hbase.client.TableDescriptor;
043import org.apache.hadoop.hbase.regionserver.HRegion;
044import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles;
045import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
046import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
047import org.apache.yetus.audience.InterfaceAudience;
048import org.apache.yetus.audience.InterfaceStability;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
053import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
054import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
055import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
056import org.apache.hbase.thirdparty.com.google.common.collect.Ordering;
057import org.apache.hbase.thirdparty.com.google.common.collect.TreeMultimap;
058
059/**
060 * Maintain information about a particular table.
061 */
062@InterfaceAudience.Private
063@InterfaceStability.Evolving
064public class HbckTableInfo {
065  private static final Logger LOG = LoggerFactory.getLogger(HbckTableInfo.class.getName());
066
067  private static final String TO_BE_LOADED = "to_be_loaded";
068
069  TableName tableName;
070  TreeSet<ServerName> deployedOn;
071
072  // backwards regions
073  final List<HbckRegionInfo> backwards = new ArrayList<>();
074
075  // sidelined big overlapped regions
076  final Map<Path, HbckRegionInfo> sidelinedRegions = new HashMap<>();
077
078  // region split calculator
079  final RegionSplitCalculator<HbckRegionInfo> sc =
080      new RegionSplitCalculator<>(HbckRegionInfo.COMPARATOR);
081
082  // Histogram of different TableDescriptors found.  Ideally there is only one!
083  final Set<TableDescriptor> htds = new HashSet<>();
084
085  // key = start split, values = set of splits in problem group
086  final Multimap<byte[], HbckRegionInfo> overlapGroups =
087      TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, HbckRegionInfo.COMPARATOR);
088
089  // list of regions derived from meta entries.
090  private ImmutableList<RegionInfo> regionsFromMeta = null;
091
092  HBaseFsck hbck;
093
094  HbckTableInfo(TableName name, HBaseFsck hbck) {
095    this.tableName = name;
096    this.hbck = hbck;
097    deployedOn = new TreeSet<>();
098  }
099
100  /**
101   * @return descriptor common to all regions.  null if are none or multiple!
102   */
103  TableDescriptor getTableDescriptor() {
104    if (htds.size() == 1) {
105      return (TableDescriptor)htds.toArray()[0];
106    } else {
107      LOG.error("None/Multiple table descriptors found for table '"
108          + tableName + "' regions: " + htds);
109    }
110    return null;
111  }
112
113  public void addRegionInfo(HbckRegionInfo hir) {
114    if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
115      // end key is absolute end key, just add it.
116      // ignore replicas other than primary for these checks
117      if (hir.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
118        sc.add(hir);
119      }
120      return;
121    }
122
123    // if not the absolute end key, check for cycle
124    if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
125      hbck.getErrors().reportError(HbckErrorReporter.ERROR_CODE.REGION_CYCLE, String.format(
126          "The endkey for this region comes before the " + "startkey, startkey=%s, endkey=%s",
127          Bytes.toStringBinary(hir.getStartKey()), Bytes.toStringBinary(hir.getEndKey())), this,
128          hir);
129      backwards.add(hir);
130      return;
131    }
132
133    // main case, add to split calculator
134    // ignore replicas other than primary for these checks
135    if (hir.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
136      sc.add(hir);
137    }
138  }
139
140  public void addServer(ServerName server) {
141    this.deployedOn.add(server);
142  }
143
144  public TableName getName() {
145    return tableName;
146  }
147
148  public int getNumRegions() {
149    return sc.getStarts().size() + backwards.size();
150  }
151
152  public synchronized ImmutableList<RegionInfo> getRegionsFromMeta(
153      TreeMap<String, HbckRegionInfo> regionInfoMap) {
154    // lazy loaded, synchronized to ensure a single load
155    if (regionsFromMeta == null) {
156      List<RegionInfo> regions = new ArrayList<>();
157      for (HbckRegionInfo h : regionInfoMap.values()) {
158        if (tableName.equals(h.getTableName())) {
159          if (h.getMetaEntry() != null) {
160            regions.add(h.getMetaEntry());
161          }
162        }
163      }
164      regionsFromMeta = Ordering.from(RegionInfo.COMPARATOR).immutableSortedCopy(regions);
165    }
166
167    return regionsFromMeta;
168  }
169
170  class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
171    HbckErrorReporter errors;
172
173    IntegrityFixSuggester(HbckTableInfo ti, HbckErrorReporter errors) {
174      this.errors = errors;
175      setTableInfo(ti);
176    }
177
178    @Override
179    public void handleRegionStartKeyNotEmpty(HbckRegionInfo hi) throws IOException {
180      errors.reportError(HbckErrorReporter.ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
181          "First region should start with an empty key.  You need to "
182              + " create a new region and regioninfo in HDFS to plug the hole.",
183          getTableInfo(), hi);
184    }
185
186    @Override
187    public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
188      errors.reportError(HbckErrorReporter.ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
189          "Last region should end with an empty key. You need to "
190              + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
191    }
192
193    @Override
194    public void handleDegenerateRegion(HbckRegionInfo hi) throws IOException{
195      errors.reportError(HbckErrorReporter.ERROR_CODE.DEGENERATE_REGION,
196          "Region has the same start and end key.", getTableInfo(), hi);
197    }
198
199    @Override
200    public void handleDuplicateStartKeys(HbckRegionInfo r1, HbckRegionInfo r2) throws IOException {
201      byte[] key = r1.getStartKey();
202      // dup start key
203      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
204          "Multiple regions have the same startkey: " + Bytes.toStringBinary(key), getTableInfo(),
205          r1);
206      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
207          "Multiple regions have the same startkey: " + Bytes.toStringBinary(key), getTableInfo(),
208          r2);
209    }
210
211    @Override
212    public void handleSplit(HbckRegionInfo r1, HbckRegionInfo r2) throws IOException{
213      byte[] key = r1.getStartKey();
214      // dup start key
215      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_ENDKEYS,
216          "Multiple regions have the same regionID: "
217              + Bytes.toStringBinary(key), getTableInfo(), r1);
218      errors.reportError(HbckErrorReporter.ERROR_CODE.DUPE_ENDKEYS,
219          "Multiple regions have the same regionID: "
220              + Bytes.toStringBinary(key), getTableInfo(), r2);
221    }
222
223    @Override
224    public void handleOverlapInRegionChain(HbckRegionInfo hi1, HbckRegionInfo hi2)
225        throws IOException {
226      errors.reportError(HbckErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
227          "There is an overlap in the region chain.", getTableInfo(), hi1, hi2);
228    }
229
230    @Override
231    public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException {
232      errors.reportError(
233          HbckErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN,
234          "There is a hole in the region chain between "
235              + Bytes.toStringBinary(holeStart) + " and "
236              + Bytes.toStringBinary(holeStop)
237              + ".  You need to create a new .regioninfo and region "
238              + "dir in hdfs to plug the hole.");
239    }
240  }
241
242  /**
243   * This handler fixes integrity errors from hdfs information.  There are
244   * basically three classes of integrity problems 1) holes, 2) overlaps, and
245   * 3) invalid regions.
246   *
247   * This class overrides methods that fix holes and the overlap group case.
248   * Individual cases of particular overlaps are handled by the general
249   * overlap group merge repair case.
250   *
251   * If hbase is online, this forces regions offline before doing merge
252   * operations.
253   */
254  class HDFSIntegrityFixer extends IntegrityFixSuggester {
255    Configuration conf;
256
257    boolean fixOverlaps = true;
258
259    HDFSIntegrityFixer(HbckTableInfo ti, HbckErrorReporter errors, Configuration conf,
260        boolean fixHoles, boolean fixOverlaps) {
261      super(ti, errors);
262      this.conf = conf;
263      this.fixOverlaps = fixOverlaps;
264      // TODO properly use fixHoles
265    }
266
267    /**
268     * This is a special case hole -- when the first region of a table is
269     * missing from META, HBase doesn't acknowledge the existance of the
270     * table.
271     */
272    @Override
273    public void handleRegionStartKeyNotEmpty(HbckRegionInfo next) throws IOException {
274      errors.reportError(HbckErrorReporter.ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
275          "First region should start with an empty key.  Creating a new " +
276              "region and regioninfo in HDFS to plug the hole.",
277          getTableInfo(), next);
278      TableDescriptor htd = getTableInfo().getTableDescriptor();
279      // from special EMPTY_START_ROW to next region's startKey
280      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName())
281          .setStartKey(HConstants.EMPTY_START_ROW)
282          .setEndKey(next.getStartKey())
283          .build();
284
285      // TODO test
286      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
287      LOG.info("Table region start key was not empty.  Created new empty region: "
288          + newRegion + " " +region);
289      hbck.fixes++;
290    }
291
292    @Override
293    public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
294      errors.reportError(HbckErrorReporter.ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
295          "Last region should end with an empty key.  Creating a new "
296              + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
297      TableDescriptor htd = getTableInfo().getTableDescriptor();
298      // from curEndKey to EMPTY_START_ROW
299      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName())
300          .setStartKey(curEndKey)
301          .setEndKey(HConstants.EMPTY_START_ROW)
302          .build();
303
304      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
305      LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
306          + " " + region);
307      hbck.fixes++;
308    }
309
310    /**
311     * There is a hole in the hdfs regions that violates the table integrity
312     * rules.  Create a new empty region that patches the hole.
313     */
314    @Override
315    public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey)
316        throws IOException {
317      errors.reportError(HbckErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN,
318          "There is a hole in the region chain between " + Bytes.toStringBinary(holeStartKey) +
319              " and " + Bytes.toStringBinary(holeStopKey) +
320              ".  Creating a new regioninfo and region " + "dir in hdfs to plug the hole.");
321      TableDescriptor htd = getTableInfo().getTableDescriptor();
322      RegionInfo newRegion =
323          RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(holeStartKey)
324              .setEndKey(holeStopKey).build();
325      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
326      LOG.info("Plugged hole by creating new empty region: " + newRegion + " " + region);
327      hbck.fixes++;
328    }
329
330    /**
331     * This takes set of overlapping regions and merges them into a single
332     * region.  This covers cases like degenerate regions, shared start key,
333     * general overlaps, duplicate ranges, and partial overlapping regions.
334     *
335     * Cases:
336     * - Clean regions that overlap
337     * - Only .oldlogs regions (can't find start/stop range, or figure out)
338     *
339     * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
340     */
341    @Override
342    public void handleOverlapGroup(Collection<HbckRegionInfo> overlap)
343        throws IOException {
344      Preconditions.checkNotNull(overlap);
345      Preconditions.checkArgument(overlap.size() >0);
346
347      if (!this.fixOverlaps) {
348        LOG.warn("Not attempting to repair overlaps.");
349        return;
350      }
351
352      if (overlap.size() > hbck.getMaxMerge()) {
353        LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
354            "regions which is greater than " + hbck.getMaxMerge() +
355            ", the max number of regions to merge");
356        if (hbck.shouldSidelineBigOverlaps()) {
357          // we only sideline big overlapped groups that exceeds the max number of regions to merge
358          sidelineBigOverlaps(overlap);
359        }
360        return;
361      }
362      if (hbck.shouldRemoveParents()) {
363        removeParentsAndFixSplits(overlap);
364      }
365      mergeOverlaps(overlap);
366    }
367
368    void removeParentsAndFixSplits(Collection<HbckRegionInfo> overlap) throws IOException {
369      Pair<byte[], byte[]> range = null;
370      HbckRegionInfo parent = null;
371      HbckRegionInfo daughterA = null;
372      HbckRegionInfo daughterB = null;
373      Collection<HbckRegionInfo> daughters = new ArrayList<HbckRegionInfo>(overlap);
374
375      String thread = Thread.currentThread().getName();
376      LOG.info("== [" + thread + "] Attempting fix splits in overlap state.");
377
378      // we only can handle a single split per group at the time
379      if (overlap.size() > 3) {
380        LOG.info("Too many overlaps were found on this group, falling back to regular merge.");
381        return;
382      }
383
384      for (HbckRegionInfo hi : overlap) {
385        if (range == null) {
386          range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
387        } else {
388          if (RegionSplitCalculator.BYTES_COMPARATOR
389              .compare(hi.getStartKey(), range.getFirst()) < 0) {
390            range.setFirst(hi.getStartKey());
391          }
392          if (RegionSplitCalculator.BYTES_COMPARATOR
393              .compare(hi.getEndKey(), range.getSecond()) > 0) {
394            range.setSecond(hi.getEndKey());
395          }
396        }
397      }
398
399      LOG.info("This group range is [" + Bytes.toStringBinary(range.getFirst()) + ", "
400          + Bytes.toStringBinary(range.getSecond()) + "]");
401
402      // attempt to find a possible parent for the edge case of a split
403      for (HbckRegionInfo hi : overlap) {
404        if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0
405            && Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
406          LOG.info("This is a parent for this group: " + hi.toString());
407          parent = hi;
408        }
409      }
410
411      // Remove parent regions from daughters collection
412      if (parent != null) {
413        daughters.remove(parent);
414      }
415
416      // Lets verify that daughters share the regionID at split time and they
417      // were created after the parent
418      for (HbckRegionInfo hi : daughters) {
419        if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0) {
420          if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
421            daughterA = hi;
422          }
423        }
424        if (Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
425          if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
426            daughterB = hi;
427          }
428        }
429      }
430
431      // daughters must share the same regionID and we should have a parent too
432      if (daughterA.getHdfsHRI().getRegionId() != daughterB.getHdfsHRI().getRegionId() ||
433          parent == null) {
434        return;
435      }
436
437      FileSystem fs = FileSystem.get(conf);
438      LOG.info("Found parent: " + parent.getRegionNameAsString());
439      LOG.info("Found potential daughter a: " + daughterA.getRegionNameAsString());
440      LOG.info("Found potential daughter b: " + daughterB.getRegionNameAsString());
441      LOG.info("Trying to fix parent in overlap by removing the parent.");
442      try {
443        hbck.closeRegion(parent);
444      } catch (IOException ioe) {
445        LOG.warn("Parent region could not be closed, continuing with regular merge...", ioe);
446        return;
447      } catch (InterruptedException ie) {
448        LOG.warn("Parent region could not be closed, continuing with regular merge...", ie);
449        return;
450      }
451
452      try {
453        hbck.offline(parent.getRegionName());
454      } catch (IOException ioe) {
455        LOG.warn("Unable to offline parent region: " + parent.getRegionNameAsString()
456            + ".  Just continuing with regular merge... ", ioe);
457        return;
458      }
459
460      try {
461        HBaseFsckRepair.removeParentInMeta(conf, parent.getHdfsHRI());
462      } catch (IOException ioe) {
463        LOG.warn("Unable to remove parent region in META: " + parent.getRegionNameAsString()
464            + ".  Just continuing with regular merge... ", ioe);
465        return;
466      }
467
468      hbck.sidelineRegionDir(fs, parent);
469      LOG.info(
470          "[" + thread + "] Sidelined parent region dir " + parent.getHdfsRegionDir() + " into " +
471              hbck.getSidelineDir());
472      hbck.debugLsr(parent.getHdfsRegionDir());
473
474      // Make sure we don't have the parents and daughters around
475      overlap.remove(parent);
476      overlap.remove(daughterA);
477      overlap.remove(daughterB);
478
479      LOG.info("Done fixing split.");
480
481    }
482
483    void mergeOverlaps(Collection<HbckRegionInfo> overlap)
484        throws IOException {
485      String thread = Thread.currentThread().getName();
486      LOG.info("== [" + thread + "] Merging regions into one region: "
487          + Joiner.on(",").join(overlap));
488      // get the min / max range and close all concerned regions
489      Pair<byte[], byte[]> range = null;
490      for (HbckRegionInfo hi : overlap) {
491        if (range == null) {
492          range = new Pair<>(hi.getStartKey(), hi.getEndKey());
493        } else {
494          if (RegionSplitCalculator.BYTES_COMPARATOR
495              .compare(hi.getStartKey(), range.getFirst()) < 0) {
496            range.setFirst(hi.getStartKey());
497          }
498          if (RegionSplitCalculator.BYTES_COMPARATOR
499              .compare(hi.getEndKey(), range.getSecond()) > 0) {
500            range.setSecond(hi.getEndKey());
501          }
502        }
503        // need to close files so delete can happen.
504        LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
505        LOG.debug("[" + thread + "] Contained region dir before close");
506        hbck.debugLsr(hi.getHdfsRegionDir());
507        try {
508          LOG.info("[" + thread + "] Closing region: " + hi);
509          hbck.closeRegion(hi);
510        } catch (IOException ioe) {
511          LOG.warn("[" + thread + "] Was unable to close region " + hi
512              + ".  Just continuing... ", ioe);
513        } catch (InterruptedException e) {
514          LOG.warn("[" + thread + "] Was unable to close region " + hi
515              + ".  Just continuing... ", e);
516        }
517
518        try {
519          LOG.info("[" + thread + "] Offlining region: " + hi);
520          hbck.offline(hi.getRegionName());
521        } catch (IOException ioe) {
522          LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
523              + ".  Just continuing... ", ioe);
524        }
525      }
526
527      // create new empty container region.
528      TableDescriptor htd = getTableInfo().getTableDescriptor();
529      // from start key to end Key
530      RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName())
531          .setStartKey(range.getFirst())
532          .setEndKey(range.getSecond())
533          .build();
534      HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
535      LOG.info("[" + thread + "] Created new empty container region: " +
536          newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
537      hbck.debugLsr(region.getRegionFileSystem().getRegionDir());
538
539      // all target regions are closed, should be able to safely cleanup.
540      boolean didFix= false;
541      Path target = region.getRegionFileSystem().getRegionDir();
542      for (HbckRegionInfo contained : overlap) {
543        LOG.info("[" + thread + "] Merging " + contained  + " into " + target);
544        int merges = hbck.mergeRegionDirs(target, contained);
545        if (merges > 0) {
546          didFix = true;
547        }
548      }
549      if (didFix) {
550        hbck.fixes++;
551      }
552    }
553
554    /**
555     * Sideline some regions in a big overlap group so that it
556     * will have fewer regions, and it is easier to merge them later on.
557     *
558     * @param bigOverlap the overlapped group with regions more than maxMerge
559     */
560    void sidelineBigOverlaps(Collection<HbckRegionInfo> bigOverlap) throws IOException {
561      int overlapsToSideline = bigOverlap.size() - hbck.getMaxMerge();
562      if (overlapsToSideline > hbck.getMaxOverlapsToSideline()) {
563        overlapsToSideline = hbck.getMaxOverlapsToSideline();
564      }
565      List<HbckRegionInfo> regionsToSideline =
566          RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
567      FileSystem fs = FileSystem.get(conf);
568      for (HbckRegionInfo regionToSideline: regionsToSideline) {
569        try {
570          LOG.info("Closing region: " + regionToSideline);
571          hbck.closeRegion(regionToSideline);
572        } catch (IOException ioe) {
573          LOG.warn("Was unable to close region " + regionToSideline
574              + ".  Just continuing... ", ioe);
575        } catch (InterruptedException e) {
576          LOG.warn("Was unable to close region " + regionToSideline
577              + ".  Just continuing... ", e);
578        }
579
580        try {
581          LOG.info("Offlining region: " + regionToSideline);
582          hbck.offline(regionToSideline.getRegionName());
583        } catch (IOException ioe) {
584          LOG.warn("Unable to offline region from master: " + regionToSideline
585              + ".  Just continuing... ", ioe);
586        }
587
588        LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
589        Path sidelineRegionDir = hbck.sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
590        if (sidelineRegionDir != null) {
591          sidelinedRegions.put(sidelineRegionDir, regionToSideline);
592          LOG.info("After sidelined big overlapped region: "
593              + regionToSideline.getRegionNameAsString()
594              + " to " + sidelineRegionDir.toString());
595          hbck.fixes++;
596        }
597      }
598    }
599  }
600
601  /**
602   * Check the region chain (from META) of this table.  We are looking for
603   * holes, overlaps, and cycles.
604   * @return false if there are errors
605   */
606  public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
607    // When table is disabled no need to check for the region chain. Some of the regions
608    // accidently if deployed, this below code might report some issues like missing start
609    // or end regions or region hole in chain and may try to fix which is unwanted.
610    if (hbck.isTableDisabled(this.tableName)) {
611      return true;
612    }
613    int originalErrorsCount = hbck.getErrors().getErrorList().size();
614    Multimap<byte[], HbckRegionInfo> regions = sc.calcCoverage();
615    SortedSet<byte[]> splits = sc.getSplits();
616
617    byte[] prevKey = null;
618    byte[] problemKey = null;
619
620    if (splits.isEmpty()) {
621      // no region for this table
622      handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
623    }
624
625    for (byte[] key : splits) {
626      Collection<HbckRegionInfo> ranges = regions.get(key);
627      if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
628        for (HbckRegionInfo rng : ranges) {
629          handler.handleRegionStartKeyNotEmpty(rng);
630        }
631      }
632
633      // check for degenerate ranges
634      for (HbckRegionInfo rng : ranges) {
635        // special endkey case converts '' to null
636        byte[] endKey = rng.getEndKey();
637        endKey = (endKey.length == 0) ? null : endKey;
638        if (Bytes.equals(rng.getStartKey(),endKey)) {
639          handler.handleDegenerateRegion(rng);
640        }
641      }
642
643      if (ranges.size() == 1) {
644        // this split key is ok -- no overlap, not a hole.
645        if (problemKey != null) {
646          LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
647        }
648        problemKey = null; // fell through, no more problem.
649      } else if (ranges.size() > 1) {
650        // set the new problem key group name, if already have problem key, just
651        // keep using it.
652        if (problemKey == null) {
653          // only for overlap regions.
654          LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
655          problemKey = key;
656        }
657        overlapGroups.putAll(problemKey, ranges);
658
659        // record errors
660        ArrayList<HbckRegionInfo> subRange = new ArrayList<>(ranges);
661        //  this dumb and n^2 but this shouldn't happen often
662        for (HbckRegionInfo r1 : ranges) {
663          if (r1.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
664            continue;
665          }
666          subRange.remove(r1);
667          for (HbckRegionInfo r2 : subRange) {
668            if (r2.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
669              continue;
670            }
671            // general case of same start key
672            if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
673              handler.handleDuplicateStartKeys(r1,r2);
674            } else if (Bytes.compareTo(r1.getEndKey(), r2.getStartKey())==0 &&
675                r1.getHdfsHRI().getRegionId() == r2.getHdfsHRI().getRegionId()) {
676              LOG.info("this is a split, log to splits");
677              handler.handleSplit(r1, r2);
678            } else {
679              // overlap
680              handler.handleOverlapInRegionChain(r1, r2);
681            }
682          }
683        }
684
685      } else if (ranges.isEmpty()) {
686        if (problemKey != null) {
687          LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
688        }
689        problemKey = null;
690
691        byte[] holeStopKey = sc.getSplits().higher(key);
692        // if higher key is null we reached the top.
693        if (holeStopKey != null) {
694          // hole
695          handler.handleHoleInRegionChain(key, holeStopKey);
696        }
697      }
698      prevKey = key;
699    }
700
701    // When the last region of a table is proper and having an empty end key, 'prevKey'
702    // will be null.
703    if (prevKey != null) {
704      handler.handleRegionEndKeyNotEmpty(prevKey);
705    }
706
707    // TODO fold this into the TableIntegrityHandler
708    if (hbck.getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
709      boolean ok = handleOverlapsParallel(handler, prevKey);
710      if (!ok) {
711        return false;
712      }
713    } else {
714      for (Collection<HbckRegionInfo> overlap : overlapGroups.asMap().values()) {
715        handler.handleOverlapGroup(overlap);
716      }
717    }
718
719    if (hbck.shouldDisplayFullReport()) {
720      // do full region split map dump
721      hbck.getErrors().print("---- Table '"  +  this.tableName
722          + "': region split map");
723      dump(splits, regions);
724      hbck.getErrors().print("---- Table '"  +  this.tableName
725          + "': overlap groups");
726      dumpOverlapProblems(overlapGroups);
727      hbck.getErrors().print("There are " + overlapGroups.keySet().size()
728          + " overlap groups with " + overlapGroups.size()
729          + " overlapping regions");
730    }
731    if (!sidelinedRegions.isEmpty()) {
732      LOG.warn("Sidelined big overlapped regions, please bulk load them!");
733      hbck.getErrors().print("---- Table '"  +  this.tableName
734          + "': sidelined big overlapped regions");
735      dumpSidelinedRegions(sidelinedRegions);
736    }
737    return hbck.getErrors().getErrorList().size() == originalErrorsCount;
738  }
739
740  private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
741      throws IOException {
742    // we parallelize overlap handler for the case we have lots of groups to fix.  We can
743    // safely assume each group is independent.
744    List<HBaseFsck.WorkItemOverlapMerge> merges = new ArrayList<>(overlapGroups.size());
745    List<Future<Void>> rets;
746    for (Collection<HbckRegionInfo> overlap : overlapGroups.asMap().values()) {
747      //
748      merges.add(new HBaseFsck.WorkItemOverlapMerge(overlap, handler));
749    }
750    try {
751      rets = hbck.executor.invokeAll(merges);
752    } catch (InterruptedException e) {
753      LOG.error("Overlap merges were interrupted", e);
754      return false;
755    }
756    for(int i=0; i<merges.size(); i++) {
757      HBaseFsck.WorkItemOverlapMerge work = merges.get(i);
758      Future<Void> f = rets.get(i);
759      try {
760        f.get();
761      } catch(ExecutionException e) {
762        LOG.warn("Failed to merge overlap group" + work, e.getCause());
763      } catch (InterruptedException e) {
764        LOG.error("Waiting for overlap merges was interrupted", e);
765        return false;
766      }
767    }
768    return true;
769  }
770
771  /**
772   * This dumps data in a visually reasonable way for visual debugging
773   */
774  private void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckRegionInfo> regions) {
775    // we display this way because the last end key should be displayed as well.
776    StringBuilder sb = new StringBuilder();
777    for (byte[] k : splits) {
778      sb.setLength(0); // clear out existing buffer, if any.
779      sb.append(Bytes.toStringBinary(k) + ":\t");
780      for (HbckRegionInfo r : regions.get(k)) {
781        sb.append("[ "+ r.toString() + ", "
782            + Bytes.toStringBinary(r.getEndKey())+ "]\t");
783      }
784      hbck.getErrors().print(sb.toString());
785    }
786  }
787
788  private void dumpOverlapProblems(Multimap<byte[], HbckRegionInfo> regions) {
789    // we display this way because the last end key should be displayed as
790    // well.
791    for (byte[] k : regions.keySet()) {
792      hbck.getErrors().print(Bytes.toStringBinary(k) + ":");
793      for (HbckRegionInfo r : regions.get(k)) {
794        hbck.getErrors().print("[ " + r.toString() + ", "
795            + Bytes.toStringBinary(r.getEndKey()) + "]");
796      }
797      hbck.getErrors().print("----");
798    }
799  }
800
801  private void dumpSidelinedRegions(Map<Path, HbckRegionInfo> regions) {
802    for (Map.Entry<Path, HbckRegionInfo> entry : regions.entrySet()) {
803      TableName tableName = entry.getValue().getTableName();
804      Path path = entry.getKey();
805      hbck.getErrors().print("This sidelined region dir should be bulk loaded: " + path.toString());
806      hbck.getErrors().print("Bulk load command looks like: " + LoadIncrementalHFiles.NAME + " " +
807          path.toUri().getPath() + " " + tableName);
808    }
809  }
810}