001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.List;
023import java.util.stream.Collectors;
024import org.apache.hadoop.hbase.CatalogFamilyFormat;
025import org.apache.hadoop.hbase.ClientMetaTableAccessor;
026import org.apache.hadoop.hbase.HRegionLocation;
027import org.apache.hadoop.hbase.MetaTableAccessor;
028import org.apache.hadoop.hbase.RegionLocations;
029import org.apache.hadoop.hbase.ServerName;
030import org.apache.hadoop.hbase.client.Connection;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.Result;
033import org.apache.hadoop.hbase.master.RegionState;
034import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
035import org.apache.hadoop.hbase.master.assignment.RegionStateStore;
036import org.apache.yetus.audience.InterfaceAudience;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 * Acts like the super class in all cases except when no Regions found in the current Master
042 * in-memory context. In this latter case, when the call to super#getRegionsOnCrashedServer returns
043 * nothing, this SCP will scan hbase:meta for references to the passed ServerName. If any found,
044 * we'll clean them up.
045 * <p>
046 * This version of SCP is for external invocation as part of fix-up (e.g. HBCK2's
047 * scheduleRecoveries); the super class is used during normal recovery operations. It is for the
048 * case where meta has references to 'Unknown Servers', servers that are in hbase:meta but not in
049 * live-server or dead-server lists; i.e. Master and hbase:meta content have deviated. It should
050 * never happen in normal running cluster but if we do drop accounting of servers, we need a means
051 * of fix-up. Eventually, as part of normal CatalogJanitor task, rather than just identify these
052 * 'Unknown Servers', it would make repair, queuing something like this HBCKSCP to do cleanup,
053 * reassigning them so Master and hbase:meta are aligned again.
054 * <p>
055 * NOTE that this SCP is costly to run; does a full scan of hbase:meta.
056 * </p>
057 */
058@InterfaceAudience.Private
059public class HBCKServerCrashProcedure extends ServerCrashProcedure {
060  private static final Logger LOG = LoggerFactory.getLogger(HBCKServerCrashProcedure.class);
061
062  /**
063   * @param serverName     Name of the crashed server.
064   * @param shouldSplitWal True if we should split WALs as part of crashed server processing.
065   * @param carryingMeta   True if carrying hbase:meta table region.
066   */
067  public HBCKServerCrashProcedure(final MasterProcedureEnv env, final ServerName serverName,
068    final boolean shouldSplitWal, final boolean carryingMeta) {
069    super(env, serverName, shouldSplitWal, carryingMeta);
070  }
071
072  /**
073   * Used when deserializing from a procedure store; we'll construct one of these then call
074   * #deserializeStateData(InputStream). Do not use directly.
075   */
076  public HBCKServerCrashProcedure() {
077  }
078
079  /**
080   * If no Regions found in Master context, then we will search hbase:meta for references to the
081   * passed server. Operator may have passed ServerName because they have found references to
082   * 'Unknown Servers'. They are using HBCKSCP to clear them out.
083   */
084  @Override
085  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NP_NULL_ON_SOME_PATH_EXCEPTION",
086      justification = "FindBugs seems confused on ps in below.")
087  List<RegionInfo> getRegionsOnCrashedServer(MasterProcedureEnv env) {
088    // Super will return an immutable list (empty if nothing on this server).
089    List<RegionInfo> ris = super.getRegionsOnCrashedServer(env);
090    if (!ris.isEmpty()) {
091      return ris;
092    }
093    // Nothing in in-master context. Check for Unknown Server! in hbase:meta.
094    // If super list is empty, then allow that an operator scheduled an SCP because they are trying
095    // to purge 'Unknown Servers' -- servers that are neither online nor in dead servers
096    // list but that ARE in hbase:meta and so showing as unknown in places like 'HBCK Report'.
097    // This mis-accounting does not happen in normal circumstance but may arise in-extremis
098    // when cluster has been damaged in operation.
099    UnknownServerVisitor visitor =
100      new UnknownServerVisitor(env.getMasterServices().getConnection(), getServerName());
101    try {
102      MetaTableAccessor.scanMetaForTableRegions(env.getMasterServices().getConnection(), visitor,
103        null);
104    } catch (IOException ioe) {
105      LOG.warn("Failed scan of hbase:meta for 'Unknown Servers'", ioe);
106      return ris;
107    }
108    // create the server state node too
109    env.getAssignmentManager().getRegionStates().createServer(getServerName());
110    LOG.info("Found {} mentions of {} in hbase:meta of OPEN/OPENING Regions: {}",
111      visitor.getReassigns().size(), getServerName(), visitor.getReassigns().stream()
112        .map(RegionInfo::getEncodedName).collect(Collectors.joining(",")));
113    return visitor.getReassigns();
114  }
115
116  /**
117   * Visitor for hbase:meta that 'fixes' Unknown Server issues. Collects a List of Regions to
118   * reassign as 'result'.
119   */
120  private static final class UnknownServerVisitor implements ClientMetaTableAccessor.Visitor {
121    private final List<RegionInfo> reassigns = new ArrayList<>();
122    private final ServerName unknownServerName;
123    private final Connection connection;
124
125    private UnknownServerVisitor(Connection connection, ServerName unknownServerName) {
126      this.connection = connection;
127      this.unknownServerName = unknownServerName;
128    }
129
130    @Override
131    public boolean visit(Result result) throws IOException {
132      RegionLocations rls = CatalogFamilyFormat.getRegionLocations(result);
133      if (rls == null) {
134        return true;
135      }
136      for (HRegionLocation hrl : rls.getRegionLocations()) {
137        if (hrl == null) {
138          continue;
139        }
140        if (hrl.getRegion() == null) {
141          continue;
142        }
143        if (hrl.getServerName() == null) {
144          continue;
145        }
146        if (!hrl.getServerName().equals(this.unknownServerName)) {
147          continue;
148        }
149        RegionState.State state = RegionStateStore.getRegionState(result, hrl.getRegion());
150        RegionState rs = new RegionState(hrl.getRegion(), state, hrl.getServerName());
151        if (rs.isClosing()) {
152          // Move region to CLOSED in hbase:meta.
153          LOG.info("Moving {} from CLOSING to CLOSED in hbase:meta",
154            hrl.getRegion().getRegionNameAsString());
155          try {
156            MetaTableAccessor.updateRegionState(this.connection, hrl.getRegion(),
157              RegionState.State.CLOSED);
158          } catch (IOException ioe) {
159            LOG.warn("Failed moving {} from CLOSING to CLOSED",
160              hrl.getRegion().getRegionNameAsString(), ioe);
161          }
162        } else if (rs.isOpening() || rs.isOpened()) {
163          this.reassigns.add(hrl.getRegion());
164        } else {
165          LOG.info("Passing {}", rs);
166        }
167      }
168      return true;
169    }
170
171    private List<RegionInfo> getReassigns() {
172      return this.reassigns;
173    }
174  }
175
176  /**
177   * The RegionStateNode will not have a location if a confirm of an OPEN fails. On fail, the
178   * RegionStateNode regionLocation is set to null. This is 'looser' than the test done in the
179   * superclass. The HBCKSCP has been scheduled by an operator via hbck2 probably at the behest of a
180   * report of an 'Unknown Server' in the 'HBCK Report'. Let the operators operation succeed even in
181   * case where the region location in the RegionStateNode is null.
182   */
183  @Override
184  protected boolean isMatchingRegionLocation(RegionStateNode rsn) {
185    return super.isMatchingRegionLocation(rsn) || rsn.getRegionLocation() == null;
186  }
187}