001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.List;
023import java.util.stream.Collectors;
024import org.apache.hadoop.hbase.CatalogFamilyFormat;
025import org.apache.hadoop.hbase.ClientMetaTableAccessor;
026import org.apache.hadoop.hbase.HRegionLocation;
027import org.apache.hadoop.hbase.MetaTableAccessor;
028import org.apache.hadoop.hbase.RegionLocations;
029import org.apache.hadoop.hbase.ServerName;
030import org.apache.hadoop.hbase.client.Connection;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.Result;
033import org.apache.hadoop.hbase.master.RegionState;
034import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
035import org.apache.hadoop.hbase.master.assignment.RegionStateStore;
036import org.apache.yetus.audience.InterfaceAudience;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 * Acts like the super class in all cases except when no Regions found in the current Master
042 * in-memory context. In this latter case, when the call to super#getRegionsOnCrashedServer returns
043 * nothing, this SCP will scan hbase:meta for references to the passed ServerName. If any found,
044 * we'll clean them up.
045 * <p>
046 * This version of SCP is for external invocation as part of fix-up (e.g. HBCK2's
047 * scheduleRecoveries); the super class is used during normal recovery operations. It is for the
048 * case where meta has references to 'Unknown Servers', servers that are in hbase:meta but not in
049 * live-server or dead-server lists; i.e. Master and hbase:meta content have deviated. It should
050 * never happen in normal running cluster but if we do drop accounting of servers, we need a means
051 * of fix-up. Eventually, as part of normal CatalogJanitor task, rather than just identify these
052 * 'Unknown Servers', it would make repair, queuing something like this HBCKSCP to do cleanup,
053 * reassigning them so Master and hbase:meta are aligned again.
054 * <p>
055 * NOTE that this SCP is costly to run; does a full scan of hbase:meta.
056 * </p>
057 */
058@InterfaceAudience.Private
059public class HBCKServerCrashProcedure extends ServerCrashProcedure {
060  private static final Logger LOG = LoggerFactory.getLogger(HBCKServerCrashProcedure.class);
061
062  /**
063   * @param serverName     Name of the crashed server.
064   * @param shouldSplitWal True if we should split WALs as part of crashed server processing.
065   * @param carryingMeta   True if carrying hbase:meta table region.
066   */
067  public HBCKServerCrashProcedure(final MasterProcedureEnv env, final ServerName serverName,
068    final boolean shouldSplitWal, final boolean carryingMeta) {
069    super(env, serverName, shouldSplitWal, carryingMeta);
070  }
071
072  /**
073   * Used when deserializing from a procedure store; we'll construct one of these then call
074   * #deserializeStateData(InputStream). Do not use directly.
075   */
076  public HBCKServerCrashProcedure() {
077  }
078
079  /**
080   * If no Regions found in Master context, then we will search hbase:meta for references to the
081   * passed server. Operator may have passed ServerName because they have found references to
082   * 'Unknown Servers'. They are using HBCKSCP to clear them out.
083   */
084  @Override
085  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NP_NULL_ON_SOME_PATH_EXCEPTION",
086      justification = "FindBugs seems confused on ps in below.")
087  List<RegionInfo> getRegionsOnCrashedServer(MasterProcedureEnv env) {
088    // Super will return an immutable list (empty if nothing on this server).
089    List<RegionInfo> ris = super.getRegionsOnCrashedServer(env);
090    if (!ris.isEmpty()) {
091      return ris;
092    }
093    // Nothing in in-master context. Check for Unknown Server! in hbase:meta.
094    // If super list is empty, then allow that an operator scheduled an SCP because they are trying
095    // to purge 'Unknown Servers' -- servers that are neither online nor in dead servers
096    // list but that ARE in hbase:meta and so showing as unknown in places like 'HBCK Report'.
097    // This mis-accounting does not happen in normal circumstance but may arise in-extremis
098    // when cluster has been damaged in operation.
099    UnknownServerVisitor visitor =
100      new UnknownServerVisitor(env.getMasterServices().getConnection(), getServerName());
101    try {
102      MetaTableAccessor.scanMetaForTableRegions(env.getMasterServices().getConnection(), visitor,
103        null);
104    } catch (IOException ioe) {
105      LOG.warn("Failed scan of hbase:meta for 'Unknown Servers'", ioe);
106      return ris;
107    }
108    LOG.info("Found {} mentions of {} in hbase:meta of OPEN/OPENING Regions: {}",
109      visitor.getReassigns().size(), getServerName(), visitor.getReassigns().stream()
110        .map(RegionInfo::getEncodedName).collect(Collectors.joining(",")));
111    return visitor.getReassigns();
112  }
113
114  /**
115   * Visitor for hbase:meta that 'fixes' Unknown Server issues. Collects a List of Regions to
116   * reassign as 'result'.
117   */
118  private static final class UnknownServerVisitor implements ClientMetaTableAccessor.Visitor {
119    private final List<RegionInfo> reassigns = new ArrayList<>();
120    private final ServerName unknownServerName;
121    private final Connection connection;
122
123    private UnknownServerVisitor(Connection connection, ServerName unknownServerName) {
124      this.connection = connection;
125      this.unknownServerName = unknownServerName;
126    }
127
128    @Override
129    public boolean visit(Result result) throws IOException {
130      RegionLocations rls = CatalogFamilyFormat.getRegionLocations(result);
131      if (rls == null) {
132        return true;
133      }
134      for (HRegionLocation hrl : rls.getRegionLocations()) {
135        if (hrl == null) {
136          continue;
137        }
138        if (hrl.getRegion() == null) {
139          continue;
140        }
141        if (hrl.getServerName() == null) {
142          continue;
143        }
144        if (!hrl.getServerName().equals(this.unknownServerName)) {
145          continue;
146        }
147        RegionState.State state = RegionStateStore.getRegionState(result, hrl.getRegion());
148        RegionState rs = new RegionState(hrl.getRegion(), state, hrl.getServerName());
149        if (rs.isClosing()) {
150          // Move region to CLOSED in hbase:meta.
151          LOG.info("Moving {} from CLOSING to CLOSED in hbase:meta",
152            hrl.getRegion().getRegionNameAsString());
153          try {
154            MetaTableAccessor.updateRegionState(this.connection, hrl.getRegion(),
155              RegionState.State.CLOSED);
156          } catch (IOException ioe) {
157            LOG.warn("Failed moving {} from CLOSING to CLOSED",
158              hrl.getRegion().getRegionNameAsString(), ioe);
159          }
160        } else if (rs.isOpening() || rs.isOpened()) {
161          this.reassigns.add(hrl.getRegion());
162        } else {
163          LOG.info("Passing {}", rs);
164        }
165      }
166      return true;
167    }
168
169    private List<RegionInfo> getReassigns() {
170      return this.reassigns;
171    }
172  }
173
174  /**
175   * The RegionStateNode will not have a location if a confirm of an OPEN fails. On fail, the
176   * RegionStateNode regionLocation is set to null. This is 'looser' than the test done in the
177   * superclass. The HBCKSCP has been scheduled by an operator via hbck2 probably at the behest of a
178   * report of an 'Unknown Server' in the 'HBCK Report'. Let the operators operation succeed even in
179   * case where the region location in the RegionStateNode is null.
180   */
181  @Override
182  protected boolean isMatchingRegionLocation(RegionStateNode rsn) {
183    return super.isMatchingRegionLocation(rsn) || rsn.getRegionLocation() == null;
184  }
185}