001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Collections;
023import java.util.List;
024import java.util.Optional;
025import java.util.Set;
026import java.util.SortedSet;
027import java.util.TreeSet;
028import java.util.stream.Collectors;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.MetaTableAccessor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.RegionInfo;
033import org.apache.hadoop.hbase.client.RegionInfoBuilder;
034import org.apache.hadoop.hbase.exceptions.MergeRegionException;
035import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
036import org.apache.hadoop.hbase.util.Bytes;
037import org.apache.hadoop.hbase.util.Pair;
038import org.apache.yetus.audience.InterfaceAudience;
039import org.slf4j.Logger;
040import org.slf4j.LoggerFactory;
041import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
042
043
044/**
045 * Server-side fixing of bad or inconsistent state in hbase:meta.
046 * Distinct from MetaTableAccessor because {@link MetaTableAccessor} is about low-level
047 * manipulations driven by the Master. This class MetaFixer is
048 * employed by the Master and it 'knows' about holes and orphans
049 * and encapsulates their fixing on behalf of the Master.
050 */
051@InterfaceAudience.Private
052class MetaFixer {
053  private static final Logger LOG = LoggerFactory.getLogger(MetaFixer.class);
054  private static final String MAX_MERGE_COUNT_KEY = "hbase.master.metafixer.max.merge.count";
055  private static final int MAX_MERGE_COUNT_DEFAULT = 10;
056
057  private final MasterServices masterServices;
058  /**
059   * Maximum for many regions to merge at a time.
060   */
061  private final int maxMergeCount;
062
063  MetaFixer(MasterServices masterServices) {
064    this.masterServices = masterServices;
065    this.maxMergeCount = this.masterServices.getConfiguration().
066        getInt(MAX_MERGE_COUNT_KEY, MAX_MERGE_COUNT_DEFAULT);
067  }
068
069  void fix() throws IOException {
070    CatalogJanitor.Report report = this.masterServices.getCatalogJanitor().getLastReport();
071    if (report == null) {
072      LOG.info("CatalogJanitor has not generated a report yet; run 'catalogjanitor_run' in " +
073          "shell or wait until CatalogJanitor chore runs.");
074      return;
075    }
076    fixHoles(report);
077    fixOverlaps(report);
078    // Run the ReplicationBarrierCleaner here; it may clear out rep_barrier rows which
079    // can help cleaning up damaged hbase:meta.
080    this.masterServices.runReplicationBarrierCleaner();
081  }
082
083  /**
084   * If hole, it papers it over by adding a region in the filesystem and to hbase:meta.
085   * Does not assign.
086   */
087  void fixHoles(CatalogJanitor.Report report) {
088    final List<Pair<RegionInfo, RegionInfo>> holes = report.getHoles();
089    if (holes.isEmpty()) {
090      LOG.info("CatalogJanitor Report contains no holes to fix. Skipping.");
091      return;
092    }
093
094    LOG.info("Identified {} region holes to fix. Detailed fixup progress logged at DEBUG.",
095      holes.size());
096
097    final List<RegionInfo> newRegionInfos = createRegionInfosForHoles(holes);
098    final List<RegionInfo> newMetaEntries = createMetaEntries(masterServices, newRegionInfos);
099    final TransitRegionStateProcedure[] assignProcedures = masterServices
100      .getAssignmentManager()
101      .createRoundRobinAssignProcedures(newMetaEntries);
102
103    masterServices.getMasterProcedureExecutor().submitProcedures(assignProcedures);
104    LOG.info(
105      "Scheduled {}/{} new regions for assignment.", assignProcedures.length, holes.size());
106  }
107
108  /**
109   * Create a new {@link RegionInfo} corresponding to each provided "hole" pair.
110   */
111  private static List<RegionInfo> createRegionInfosForHoles(
112    final List<Pair<RegionInfo, RegionInfo>> holes) {
113    final List<RegionInfo> newRegionInfos = holes.stream()
114      .map(MetaFixer::getHoleCover)
115      .filter(Optional::isPresent)
116      .map(Optional::get)
117      .collect(Collectors.toList());
118    LOG.debug("Constructed {}/{} RegionInfo descriptors corresponding to identified holes.",
119      newRegionInfos.size(), holes.size());
120    return newRegionInfos;
121  }
122
123  /**
124   * @return Attempts to calculate a new {@link RegionInfo} that covers the region range described
125   *   in {@code hole}.
126   */
127  private static Optional<RegionInfo> getHoleCover(Pair<RegionInfo, RegionInfo> hole) {
128    final RegionInfo left = hole.getFirst();
129    final RegionInfo right = hole.getSecond();
130
131    if (left.getTable().equals(right.getTable())) {
132      // Simple case.
133      if (Bytes.compareTo(left.getEndKey(), right.getStartKey()) >= 0) {
134        LOG.warn("Skipping hole fix; left-side endKey is not less than right-side startKey;"
135          + " left=<{}>, right=<{}>", left, right);
136        return Optional.empty();
137      }
138      return Optional.of(buildRegionInfo(left.getTable(), left.getEndKey(), right.getStartKey()));
139    }
140
141    final boolean leftUndefined = left.equals(RegionInfo.UNDEFINED);
142    final boolean rightUndefined = right.equals(RegionInfo.UNDEFINED);
143    final boolean last = left.isLast();
144    final boolean first = right.isFirst();
145    if (leftUndefined && rightUndefined) {
146      LOG.warn("Skipping hole fix; both the hole left-side and right-side RegionInfos are " +
147        "UNDEFINED; left=<{}>, right=<{}>", left, right);
148      return Optional.empty();
149    }
150    if (leftUndefined || last) {
151      return Optional.of(
152        buildRegionInfo(right.getTable(), HConstants.EMPTY_START_ROW, right.getStartKey()));
153    }
154    if (rightUndefined || first) {
155      return Optional.of(
156        buildRegionInfo(left.getTable(), left.getEndKey(), HConstants.EMPTY_END_ROW));
157    }
158    LOG.warn("Skipping hole fix; don't know what to do with left=<{}>, right=<{}>", left, right);
159    return Optional.empty();
160  }
161
162  private static RegionInfo buildRegionInfo(TableName tn, byte [] start, byte [] end) {
163    return RegionInfoBuilder.newBuilder(tn).setStartKey(start).setEndKey(end).build();
164  }
165
166  /**
167   * Create entries in the {@code hbase:meta} for each provided {@link RegionInfo}. Best effort.
168   * @param masterServices used to connect to {@code hbase:meta}
169   * @param newRegionInfos the new {@link RegionInfo} entries to add to the filesystem
170   * @return a list of {@link RegionInfo} entries for which {@code hbase:meta} entries were
171   *   successfully created
172   */
173  private static List<RegionInfo> createMetaEntries(final MasterServices masterServices,
174    final List<RegionInfo> newRegionInfos) {
175
176    final List<Either<RegionInfo, IOException>> addMetaEntriesResults = newRegionInfos.stream()
177      .map(regionInfo -> {
178        try {
179          MetaTableAccessor.addRegionToMeta(masterServices.getConnection(), regionInfo);
180          masterServices.getAssignmentManager()
181            .getRegionStates()
182            .updateRegionState(regionInfo, RegionState.State.CLOSED);
183          return Either.<RegionInfo, IOException>ofLeft(regionInfo);
184        } catch (IOException e) {
185          return Either.<RegionInfo, IOException>ofRight(e);
186        }
187      })
188      .collect(Collectors.toList());
189    final List<RegionInfo> createMetaEntriesSuccesses = addMetaEntriesResults.stream()
190      .filter(Either::hasLeft)
191      .map(Either::getLeft)
192      .collect(Collectors.toList());
193    final List<IOException> createMetaEntriesFailures = addMetaEntriesResults.stream()
194      .filter(Either::hasRight)
195      .map(Either::getRight)
196      .collect(Collectors.toList());
197    LOG.debug("Added {}/{} entries to hbase:meta",
198      createMetaEntriesSuccesses.size(), newRegionInfos.size());
199
200    if (!createMetaEntriesFailures.isEmpty()) {
201      LOG.warn("Failed to create entries in hbase:meta for {}/{} RegionInfo descriptors. First"
202          + " failure message included; full list of failures with accompanying stack traces is"
203          + " available at log level DEBUG. message={}", createMetaEntriesFailures.size(),
204        addMetaEntriesResults.size(), createMetaEntriesFailures.get(0).getMessage());
205      if (LOG.isDebugEnabled()) {
206        createMetaEntriesFailures.forEach(
207          ioe -> LOG.debug("Attempt to fix region hole in hbase:meta failed.", ioe));
208      }
209    }
210
211    return createMetaEntriesSuccesses;
212  }
213
214  /**
215   * Fix overlaps noted in CJ consistency report.
216   */
217  void fixOverlaps(CatalogJanitor.Report report) throws IOException {
218    for (Set<RegionInfo> regions: calculateMerges(maxMergeCount, report.getOverlaps())) {
219      RegionInfo [] regionsArray = regions.toArray(new RegionInfo [] {});
220      try {
221        this.masterServices.mergeRegions(regionsArray,
222            false, HConstants.NO_NONCE, HConstants.NO_NONCE);
223      } catch (MergeRegionException mre) {
224        LOG.warn("Failed overlap fix of {}", regionsArray, mre);
225      }
226    }
227  }
228
229  /**
230   * Run through <code>overlaps</code> and return a list of merges to run.
231   * Presumes overlaps are ordered (which they are coming out of the CatalogJanitor
232   * consistency report).
233   * @param maxMergeCount Maximum regions to merge at a time (avoid merging
234   *   100k regions in one go!)
235   */
236  @VisibleForTesting
237  static List<SortedSet<RegionInfo>> calculateMerges(int maxMergeCount,
238      List<Pair<RegionInfo, RegionInfo>> overlaps) {
239    if (overlaps.isEmpty()) {
240      LOG.debug("No overlaps.");
241      return Collections.emptyList();
242    }
243    List<SortedSet<RegionInfo>> merges = new ArrayList<>();
244    SortedSet<RegionInfo> currentMergeSet = new TreeSet<>();
245    RegionInfo regionInfoWithlargestEndKey =  null;
246    for (Pair<RegionInfo, RegionInfo> pair: overlaps) {
247      if (regionInfoWithlargestEndKey != null) {
248        if (!isOverlap(regionInfoWithlargestEndKey, pair) ||
249            currentMergeSet.size() >= maxMergeCount) {
250          merges.add(currentMergeSet);
251          currentMergeSet = new TreeSet<>();
252        }
253      }
254      currentMergeSet.add(pair.getFirst());
255      currentMergeSet.add(pair.getSecond());
256      regionInfoWithlargestEndKey = getRegionInfoWithLargestEndKey(
257        getRegionInfoWithLargestEndKey(pair.getFirst(), pair.getSecond()),
258          regionInfoWithlargestEndKey);
259    }
260    merges.add(currentMergeSet);
261    return merges;
262  }
263
264  /**
265   * @return Either <code>a</code> or <code>b</code>, whichever has the
266   *   endkey that is furthest along in the Table.
267   */
268  @VisibleForTesting
269  static RegionInfo getRegionInfoWithLargestEndKey(RegionInfo a, RegionInfo b) {
270    if (a == null) {
271      // b may be null.
272      return b;
273    }
274    if (b == null) {
275      // Both are null. The return is not-defined.
276      return a;
277    }
278    if (!a.getTable().equals(b.getTable())) {
279      // This is an odd one. This should be the right answer.
280      return b;
281    }
282    if (a.isLast()) {
283      return a;
284    }
285    if (b.isLast()) {
286      return b;
287    }
288    int compare = Bytes.compareTo(a.getEndKey(), b.getEndKey());
289    return compare == 0 || compare > 0? a: b;
290  }
291
292  /**
293   * @return True if an overlap found between passed in <code>ri</code> and
294   *   the <code>pair</code>. Does NOT check the pairs themselves overlap.
295   */
296  @VisibleForTesting
297  static boolean isOverlap(RegionInfo ri, Pair<RegionInfo, RegionInfo> pair) {
298    if (ri == null || pair == null) {
299      // Can't be an overlap in either of these cases.
300      return false;
301    }
302    return ri.isOverlap(pair.getFirst()) || ri.isOverlap(pair.getSecond());
303  }
304
305  /**
306   * A union over {@link L} and {@link R}.
307   */
308  private static class Either<L, R> {
309    private final L left;
310    private final R right;
311
312    public static <L, R> Either<L, R> ofLeft(L left) {
313      return new Either<>(left, null);
314    }
315
316    public static <L, R> Either<L, R> ofRight(R right) {
317      return new Either<>(null, right);
318    }
319
320    Either(L left, R right) {
321      this.left = left;
322      this.right = right;
323    }
324
325    public boolean hasLeft() {
326      return left != null;
327    }
328
329    public L getLeft() {
330      if (!hasLeft()) {
331        throw new IllegalStateException("Either contains no left.");
332      }
333      return left;
334    }
335
336    public boolean hasRight() {
337      return right != null;
338    }
339
340    public R getRight() {
341      if (!hasRight()) {
342        throw new IllegalStateException("Either contains no right.");
343      }
344      return right;
345    }
346  }
347}