001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Collections;
023import java.util.List;
024import java.util.Set;
025import java.util.SortedSet;
026import java.util.TreeSet;
027
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.MetaTableAccessor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.RegionInfo;
033import org.apache.hadoop.hbase.client.RegionInfoBuilder;
034import org.apache.hadoop.hbase.exceptions.MergeRegionException;
035import org.apache.hadoop.hbase.regionserver.HRegion;
036import org.apache.hadoop.hbase.util.Bytes;
037import org.apache.hadoop.hbase.util.FSUtils;
038import org.apache.hadoop.hbase.util.Pair;
039import org.apache.yetus.audience.InterfaceAudience;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043
044import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
045
046
047/**
048 * Server-side fixing of bad or inconsistent state in hbase:meta.
049 * Distinct from MetaTableAccessor because {@link MetaTableAccessor} is about low-level
050 * manipulations driven by the Master. This class MetaFixer is
051 * employed by the Master and it 'knows' about holes and orphans
052 * and encapsulates their fixing on behalf of the Master.
053 */
054@InterfaceAudience.Private
055class MetaFixer {
056  private static final Logger LOG = LoggerFactory.getLogger(MetaFixer.class);
057  private static final String MAX_MERGE_COUNT_KEY = "hbase.master.metafixer.max.merge.count";
058  private static final int MAX_MERGE_COUNT_DEFAULT = 10;
059  private final MasterServices masterServices;
060  /**
061   * Maximum for many regions to merge at a time.
062   */
063  private final int maxMergeCount;
064
065  MetaFixer(MasterServices masterServices) {
066    this.masterServices = masterServices;
067    this.maxMergeCount = this.masterServices.getConfiguration().
068        getInt(MAX_MERGE_COUNT_KEY, MAX_MERGE_COUNT_DEFAULT);
069  }
070
071  void fix() throws IOException {
072    CatalogJanitor.Report report = this.masterServices.getCatalogJanitor().getLastReport();
073    if (report == null) {
074      LOG.info("CatalogJanitor has not generated a report yet; run 'catalogjanitor_run' in " +
075          "shell or wait until CatalogJanitor chore runs.");
076      return;
077    }
078    fixHoles(report);
079    fixOverlaps(report);
080    // Run the ReplicationBarrierCleaner here; it may clear out rep_barrier rows which
081    // can help cleaning up damaged hbase:meta.
082    this.masterServices.runReplicationBarrierCleaner();
083  }
084
085  /**
086   * If hole, it papers it over by adding a region in the filesystem and to hbase:meta.
087   * Does not assign.
088   */
089  void fixHoles(CatalogJanitor.Report report) throws IOException {
090    List<Pair<RegionInfo, RegionInfo>> holes = report.getHoles();
091    if (holes.isEmpty()) {
092      LOG.debug("No holes.");
093      return;
094    }
095    for (Pair<RegionInfo, RegionInfo> p: holes) {
096      RegionInfo ri = getHoleCover(p);
097      if (ri == null) {
098        continue;
099      }
100      Configuration configuration = this.masterServices.getConfiguration();
101      HRegion.createRegionDir(configuration, ri, FSUtils.getRootDir(configuration));
102      // If an error here, then we'll have a region in the filesystem but not
103      // in hbase:meta (if the below fails). Should be able to rerun the fix.
104      // Add to hbase:meta and then update in-memory state so it knows of new
105      // Region; addRegionToMeta adds region and adds a state column set to CLOSED.
106      MetaTableAccessor.addRegionToMeta(this.masterServices.getConnection(), ri);
107      this.masterServices.getAssignmentManager().getRegionStates().
108          updateRegionState(ri, RegionState.State.CLOSED);
109      LOG.info("Fixed hole by adding {} in CLOSED state; region NOT assigned (assign to ONLINE).",
110          ri);
111    }
112  }
113
114  /**
115   * @return Calculated RegionInfo that covers the hole <code>hole</code>
116   */
117  private RegionInfo getHoleCover(Pair<RegionInfo, RegionInfo> hole) {
118    RegionInfo holeCover = null;
119    RegionInfo left = hole.getFirst();
120    RegionInfo right = hole.getSecond();
121    if (left.getTable().equals(right.getTable())) {
122      // Simple case.
123      if (Bytes.compareTo(left.getEndKey(), right.getStartKey()) >= 0) {
124        LOG.warn("Skipping hole fix; left-side endKey is not less than right-side startKey; " +
125            "left=<{}>, right=<{}>", left, right);
126        return holeCover;
127      }
128      holeCover = buildRegionInfo(left.getTable(), left.getEndKey(), right.getStartKey());
129    } else {
130      boolean leftUndefined = left.equals(RegionInfo.UNDEFINED);
131      boolean rightUnefined = right.equals(RegionInfo.UNDEFINED);
132      boolean last = left.isLast();
133      boolean first = right.isFirst();
134      if (leftUndefined && rightUnefined) {
135        LOG.warn("Skipping hole fix; both the hole left-side and right-side RegionInfos are " +
136            "UNDEFINED; left=<{}>, right=<{}>", left, right);
137        return holeCover;
138      }
139      if (leftUndefined || last) {
140        holeCover = buildRegionInfo(right.getTable(), HConstants.EMPTY_START_ROW,
141            right.getStartKey());
142      } else if (rightUnefined || first) {
143        holeCover = buildRegionInfo(left.getTable(), left.getEndKey(), HConstants.EMPTY_END_ROW);
144      } else {
145        LOG.warn("Skipping hole fix; don't know what to do with left=<{}>, right=<{}>",
146            left, right);
147        return holeCover;
148      }
149    }
150    return holeCover;
151  }
152
153  private RegionInfo buildRegionInfo(TableName tn, byte [] start, byte [] end) {
154    return RegionInfoBuilder.newBuilder(tn).setStartKey(start).setEndKey(end).build();
155  }
156
157  /**
158   * Fix overlaps noted in CJ consistency report.
159   */
160  void fixOverlaps(CatalogJanitor.Report report) throws IOException {
161    for (Set<RegionInfo> regions: calculateMerges(maxMergeCount, report.getOverlaps())) {
162      RegionInfo [] regionsArray = regions.toArray(new RegionInfo [] {});
163      try {
164        this.masterServices.mergeRegions(regionsArray,
165            false, HConstants.NO_NONCE, HConstants.NO_NONCE);
166      } catch (MergeRegionException mre) {
167        LOG.warn("Failed overlap fix of {}", regionsArray, mre);
168      }
169    }
170  }
171
172  /**
173   * Run through <code>overlaps</code> and return a list of merges to run.
174   * Presumes overlaps are ordered (which they are coming out of the CatalogJanitor
175   * consistency report).
176   * @param maxMergeCount Maximum regions to merge at a time (avoid merging
177   *   100k regions in one go!)
178   */
179  @VisibleForTesting
180  static List<SortedSet<RegionInfo>> calculateMerges(int maxMergeCount,
181      List<Pair<RegionInfo, RegionInfo>> overlaps) {
182    if (overlaps.isEmpty()) {
183      LOG.debug("No overlaps.");
184      return Collections.emptyList();
185    }
186    List<SortedSet<RegionInfo>> merges = new ArrayList<>();
187    SortedSet<RegionInfo> currentMergeSet = new TreeSet<>();
188    RegionInfo regionInfoWithlargestEndKey =  null;
189    for (Pair<RegionInfo, RegionInfo> pair: overlaps) {
190      if (regionInfoWithlargestEndKey != null) {
191        if (!isOverlap(regionInfoWithlargestEndKey, pair) ||
192            currentMergeSet.size() >= maxMergeCount) {
193          merges.add(currentMergeSet);
194          currentMergeSet = new TreeSet<>();
195        }
196      }
197      currentMergeSet.add(pair.getFirst());
198      currentMergeSet.add(pair.getSecond());
199      regionInfoWithlargestEndKey = getRegionInfoWithLargestEndKey(
200        getRegionInfoWithLargestEndKey(pair.getFirst(), pair.getSecond()),
201          regionInfoWithlargestEndKey);
202    }
203    merges.add(currentMergeSet);
204    return merges;
205  }
206
207  /**
208   * @return Either <code>a</code> or <code>b</code>, whichever has the
209   *   endkey that is furthest along in the Table.
210   */
211  @VisibleForTesting
212  static RegionInfo getRegionInfoWithLargestEndKey(RegionInfo a, RegionInfo b) {
213    if (a == null) {
214      // b may be null.
215      return b;
216    }
217    if (b == null) {
218      // Both are null. The return is not-defined.
219      return a;
220    }
221    if (!a.getTable().equals(b.getTable())) {
222      // This is an odd one. This should be the right answer.
223      return b;
224    }
225    if (a.isLast()) {
226      return a;
227    }
228    if (b.isLast()) {
229      return b;
230    }
231    int compare = Bytes.compareTo(a.getEndKey(), b.getEndKey());
232    return compare == 0 || compare > 0? a: b;
233  }
234
235  /**
236   * @return True if an overlap found between passed in <code>ri</code> and
237   *   the <code>pair</code>. Does NOT check the pairs themselves overlap.
238   */
239  @VisibleForTesting
240  static boolean isOverlap(RegionInfo ri, Pair<RegionInfo, RegionInfo> pair) {
241    if (ri == null || pair == null) {
242      // Can't be an overlap in either of these cases.
243      return false;
244    }
245    return ri.isOverlap(pair.getFirst()) || ri.isOverlap(pair.getSecond());
246  }
247}