001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.procedure; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.List; 023import java.util.stream.Collectors; 024import org.apache.hadoop.hbase.CatalogFamilyFormat; 025import org.apache.hadoop.hbase.ClientMetaTableAccessor; 026import org.apache.hadoop.hbase.HRegionLocation; 027import org.apache.hadoop.hbase.MetaTableAccessor; 028import org.apache.hadoop.hbase.RegionLocations; 029import org.apache.hadoop.hbase.ServerName; 030import org.apache.hadoop.hbase.client.Connection; 031import org.apache.hadoop.hbase.client.RegionInfo; 032import org.apache.hadoop.hbase.client.Result; 033import org.apache.hadoop.hbase.master.RegionState; 034import org.apache.hadoop.hbase.master.assignment.RegionStateNode; 035import org.apache.hadoop.hbase.master.assignment.RegionStateStore; 036import org.apache.yetus.audience.InterfaceAudience; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * Acts like the super class in all cases except when no Regions found in the current Master 042 * in-memory context. In this latter case, when the call to super#getRegionsOnCrashedServer returns 043 * nothing, this SCP will scan hbase:meta for references to the passed ServerName. If any found, 044 * we'll clean them up. 045 * <p> 046 * This version of SCP is for external invocation as part of fix-up (e.g. HBCK2's 047 * scheduleRecoveries); the super class is used during normal recovery operations. It is for the 048 * case where meta has references to 'Unknown Servers', servers that are in hbase:meta but not in 049 * live-server or dead-server lists; i.e. Master and hbase:meta content have deviated. It should 050 * never happen in normal running cluster but if we do drop accounting of servers, we need a means 051 * of fix-up. Eventually, as part of normal CatalogJanitor task, rather than just identify these 052 * 'Unknown Servers', it would make repair, queuing something like this HBCKSCP to do cleanup, 053 * reassigning them so Master and hbase:meta are aligned again. 054 * <p> 055 * NOTE that this SCP is costly to run; does a full scan of hbase:meta. 056 * </p> 057 */ 058@InterfaceAudience.Private 059public class HBCKServerCrashProcedure extends ServerCrashProcedure { 060 private static final Logger LOG = LoggerFactory.getLogger(HBCKServerCrashProcedure.class); 061 062 /** 063 * @param serverName Name of the crashed server. 064 * @param shouldSplitWal True if we should split WALs as part of crashed server processing. 065 * @param carryingMeta True if carrying hbase:meta table region. 066 */ 067 public HBCKServerCrashProcedure(final MasterProcedureEnv env, final ServerName serverName, 068 final boolean shouldSplitWal, final boolean carryingMeta) { 069 super(env, serverName, shouldSplitWal, carryingMeta); 070 } 071 072 /** 073 * Used when deserializing from a procedure store; we'll construct one of these then call 074 * #deserializeStateData(InputStream). Do not use directly. 075 */ 076 public HBCKServerCrashProcedure() { 077 } 078 079 /** 080 * If no Regions found in Master context, then we will search hbase:meta for references to the 081 * passed server. Operator may have passed ServerName because they have found references to 082 * 'Unknown Servers'. They are using HBCKSCP to clear them out. 083 */ 084 @Override 085 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NP_NULL_ON_SOME_PATH_EXCEPTION", 086 justification = "FindBugs seems confused on ps in below.") 087 List<RegionInfo> getRegionsOnCrashedServer(MasterProcedureEnv env) { 088 // Super will return an immutable list (empty if nothing on this server). 089 List<RegionInfo> ris = super.getRegionsOnCrashedServer(env); 090 if (!ris.isEmpty()) { 091 return ris; 092 } 093 // Nothing in in-master context. Check for Unknown Server! in hbase:meta. 094 // If super list is empty, then allow that an operator scheduled an SCP because they are trying 095 // to purge 'Unknown Servers' -- servers that are neither online nor in dead servers 096 // list but that ARE in hbase:meta and so showing as unknown in places like 'HBCK Report'. 097 // This mis-accounting does not happen in normal circumstance but may arise in-extremis 098 // when cluster has been damaged in operation. 099 UnknownServerVisitor visitor = 100 new UnknownServerVisitor(env.getMasterServices().getConnection(), getServerName()); 101 try { 102 MetaTableAccessor.scanMetaForTableRegions(env.getMasterServices().getConnection(), visitor, 103 null); 104 } catch (IOException ioe) { 105 LOG.warn("Failed scan of hbase:meta for 'Unknown Servers'", ioe); 106 return ris; 107 } 108 // create the server state node too 109 env.getAssignmentManager().getRegionStates().createServer(getServerName()); 110 LOG.info("Found {} mentions of {} in hbase:meta of OPEN/OPENING Regions: {}", 111 visitor.getReassigns().size(), getServerName(), visitor.getReassigns().stream() 112 .map(RegionInfo::getEncodedName).collect(Collectors.joining(","))); 113 return visitor.getReassigns(); 114 } 115 116 /** 117 * Visitor for hbase:meta that 'fixes' Unknown Server issues. Collects a List of Regions to 118 * reassign as 'result'. 119 */ 120 private static final class UnknownServerVisitor implements ClientMetaTableAccessor.Visitor { 121 private final List<RegionInfo> reassigns = new ArrayList<>(); 122 private final ServerName unknownServerName; 123 private final Connection connection; 124 125 private UnknownServerVisitor(Connection connection, ServerName unknownServerName) { 126 this.connection = connection; 127 this.unknownServerName = unknownServerName; 128 } 129 130 @Override 131 public boolean visit(Result result) throws IOException { 132 RegionLocations rls = CatalogFamilyFormat.getRegionLocations(result); 133 if (rls == null) { 134 return true; 135 } 136 for (HRegionLocation hrl : rls.getRegionLocations()) { 137 if (hrl == null) { 138 continue; 139 } 140 if (hrl.getRegion() == null) { 141 continue; 142 } 143 if (hrl.getServerName() == null) { 144 continue; 145 } 146 if (!hrl.getServerName().equals(this.unknownServerName)) { 147 continue; 148 } 149 RegionState.State state = RegionStateStore.getRegionState(result, hrl.getRegion()); 150 RegionState rs = new RegionState(hrl.getRegion(), state, hrl.getServerName()); 151 if (rs.isClosing()) { 152 // Move region to CLOSED in hbase:meta. 153 LOG.info("Moving {} from CLOSING to CLOSED in hbase:meta", 154 hrl.getRegion().getRegionNameAsString()); 155 try { 156 MetaTableAccessor.updateRegionState(this.connection, hrl.getRegion(), 157 RegionState.State.CLOSED); 158 } catch (IOException ioe) { 159 LOG.warn("Failed moving {} from CLOSING to CLOSED", 160 hrl.getRegion().getRegionNameAsString(), ioe); 161 } 162 } else if (rs.isOpening() || rs.isOpened()) { 163 this.reassigns.add(hrl.getRegion()); 164 } else { 165 LOG.info("Passing {}", rs); 166 } 167 } 168 return true; 169 } 170 171 private List<RegionInfo> getReassigns() { 172 return this.reassigns; 173 } 174 } 175 176 /** 177 * The RegionStateNode will not have a location if a confirm of an OPEN fails. On fail, the 178 * RegionStateNode regionLocation is set to null. This is 'looser' than the test done in the 179 * superclass. The HBCKSCP has been scheduled by an operator via hbck2 probably at the behest of a 180 * report of an 'Unknown Server' in the 'HBCK Report'. Let the operators operation succeed even in 181 * case where the region location in the RegionStateNode is null. 182 */ 183 @Override 184 protected boolean isMatchingRegionLocation(RegionStateNode rsn) { 185 return super.isMatchingRegionLocation(rsn) || rsn.getRegionLocation() == null; 186 } 187}