001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.janitor; 019 020import java.io.IOException; 021import org.apache.hadoop.hbase.HConstants; 022import org.apache.hadoop.hbase.HRegionLocation; 023import org.apache.hadoop.hbase.MetaTableAccessor; 024import org.apache.hadoop.hbase.RegionLocations; 025import org.apache.hadoop.hbase.ServerName; 026import org.apache.hadoop.hbase.client.RegionInfo; 027import org.apache.hadoop.hbase.client.RegionInfoBuilder; 028import org.apache.hadoop.hbase.client.Result; 029import org.apache.hadoop.hbase.client.TableState; 030import org.apache.hadoop.hbase.master.MasterServices; 031import org.apache.hadoop.hbase.master.RegionState; 032import org.apache.hadoop.hbase.master.ServerManager; 033import org.apache.hadoop.hbase.util.Bytes; 034import org.apache.hadoop.hbase.util.Pair; 035import org.apache.yetus.audience.InterfaceAudience; 036import org.slf4j.Logger; 037import org.slf4j.LoggerFactory; 038 039/** 040 * Visitor we use in here in CatalogJanitor to go against hbase:meta table. Generates a Report made 041 * of a collection of split parents and counts of rows in the hbase:meta table. Also runs hbase:meta 042 * consistency checks to generate more report. Report is NOT ready until after this visitor has been 043 * {@link #close()}'d. 044 */ 045@InterfaceAudience.Private 046class ReportMakingVisitor implements MetaTableAccessor.CloseableVisitor { 047 048 private static final Logger LOG = LoggerFactory.getLogger(ReportMakingVisitor.class); 049 050 private final MasterServices services; 051 private volatile boolean closed; 052 053 /** 054 * Report is not done until after the close has been called. 055 */ 056 private Report report = new Report(); 057 058 /** 059 * RegionInfo from previous row. 060 */ 061 private RegionInfo previous = null; 062 063 /** 064 * Keep account of the highest end key seen as we move through hbase:meta. Usually, the current 065 * RegionInfo has the highest end key but if an overlap, this may no longer hold. An overlap may 066 * be a region with startkey 'd' and endkey 'g'. The next region in meta may be 'e' to 'f' and 067 * then 'f' to 'g'. Looking at previous and current meta row, we won't know about the 'd' to 'g' 068 * overlap unless we keep a running 'highest-endpoint-seen'. 069 */ 070 private RegionInfo highestEndKeyRegionInfo = null; 071 072 ReportMakingVisitor(MasterServices services) { 073 this.services = services; 074 } 075 076 /** 077 * Do not call until after {@link #close()}. Will throw a {@link RuntimeException} if you do. 078 */ 079 Report getReport() { 080 if (!this.closed) { 081 throw new RuntimeException("Report not ready until after close()"); 082 } 083 return this.report; 084 } 085 086 @Override 087 public boolean visit(Result r) { 088 if (r == null || r.isEmpty()) { 089 return true; 090 } 091 this.report.count++; 092 RegionInfo regionInfo = null; 093 try { 094 regionInfo = metaTableConsistencyCheck(r); 095 } catch (Throwable t) { 096 LOG.warn("Failed consistency check on {}", Bytes.toStringBinary(r.getRow()), t); 097 } 098 if (regionInfo != null) { 099 LOG.trace(regionInfo.toString()); 100 if (regionInfo.isSplitParent()) { // splitParent means split and offline. 101 this.report.splitParents.put(regionInfo, r); 102 } 103 if (MetaTableAccessor.hasMergeRegions(r.rawCells())) { 104 this.report.mergedRegions.put(regionInfo, r); 105 } 106 } 107 // Returning true means "keep scanning" 108 return true; 109 } 110 111 /** 112 * Check row. 113 * @param metaTableRow Row from hbase:meta table. 114 * @return Returns default regioninfo found in row parse as a convenience to save on having to do 115 * a double-parse of Result. 116 */ 117 private RegionInfo metaTableConsistencyCheck(Result metaTableRow) { 118 RegionInfo ri; 119 // Locations comes back null if the RegionInfo field is empty. 120 // If locations is null, ensure the regioninfo is for sure empty before progressing. 121 // If really empty, report as missing regioninfo! Otherwise, can run server check 122 // and get RegionInfo from locations. 123 RegionLocations locations = MetaTableAccessor.getRegionLocations(metaTableRow); 124 if (locations == null) { 125 ri = MetaTableAccessor.getRegionInfo(metaTableRow, HConstants.REGIONINFO_QUALIFIER); 126 } else { 127 ri = locations.getDefaultRegionLocation().getRegion(); 128 checkServer(locations); 129 } 130 131 if (ri == null) { 132 this.report.emptyRegionInfo.add(metaTableRow.getRow()); 133 return ri; 134 } 135 136 if (!Bytes.equals(metaTableRow.getRow(), ri.getRegionName())) { 137 LOG.warn( 138 "INCONSISTENCY: Row name is not equal to serialized info:regioninfo content; " + 139 "row={} {}; See if RegionInfo is referenced in another hbase:meta row? Delete?", 140 Bytes.toStringBinary(metaTableRow.getRow()), ri.getRegionNameAsString()); 141 return null; 142 } 143 // Skip split parent region 144 if (ri.isSplitParent()) { 145 return ri; 146 } 147 // If table is disabled, skip integrity check. 148 if (!isTableDisabled(ri)) { 149 if (isTableTransition(ri)) { 150 // HBCK1 used to have a special category for missing start or end keys. 151 // We'll just lump them in as 'holes'. 152 153 // This is a table transition. If this region is not first region, report a hole. 154 if (!ri.isFirst()) { 155 addHole(RegionInfoBuilder.UNDEFINED, ri); 156 } 157 // This is a table transition. If last region was not last region of previous table, 158 // report a hole 159 if (this.previous != null && !this.previous.isLast()) { 160 addHole(this.previous, RegionInfoBuilder.UNDEFINED); 161 } 162 } else { 163 if (!this.previous.isNext(ri)) { 164 if (this.previous.isOverlap(ri)) { 165 addOverlap(this.previous, ri); 166 } else if (ri.isOverlap(this.highestEndKeyRegionInfo)) { 167 // We may have seen a region a few rows back that overlaps this one. 168 addOverlap(this.highestEndKeyRegionInfo, ri); 169 } else if (!this.highestEndKeyRegionInfo.isNext(ri)) { 170 // Need to check the case if this.highestEndKeyRegionInfo.isNext(ri). If no, 171 // report a hole, otherwise, it is ok. For an example, 172 // previous: [aa, bb), ri: [cc, dd), highestEndKeyRegionInfo: [a, cc) 173 // In this case, it should not report a hole, as highestEndKeyRegionInfo covers 174 // the hole between previous and ri. 175 addHole(this.previous, ri); 176 } 177 } else if (ri.isOverlap(this.highestEndKeyRegionInfo)) { 178 // We may have seen a region a few rows back that overlaps this one 179 // even though it properly 'follows' the region just before. 180 addOverlap(this.highestEndKeyRegionInfo, ri); 181 } 182 } 183 } 184 this.previous = ri; 185 this.highestEndKeyRegionInfo = 186 MetaFixer.getRegionInfoWithLargestEndKey(this.highestEndKeyRegionInfo, ri); 187 return ri; 188 } 189 190 private void addOverlap(RegionInfo a, RegionInfo b) { 191 this.report.overlaps.add(new Pair<>(a, b)); 192 } 193 194 private void addHole(RegionInfo a, RegionInfo b) { 195 this.report.holes.add(new Pair<>(a, b)); 196 } 197 198 /** 199 * @return True if table is disabled or disabling; defaults false! 200 */ 201 boolean isTableDisabled(RegionInfo ri) { 202 if (ri == null) { 203 return false; 204 } 205 if (this.services == null) { 206 return false; 207 } 208 if (this.services.getTableStateManager() == null) { 209 return false; 210 } 211 TableState state = null; 212 try { 213 state = this.services.getTableStateManager().getTableState(ri.getTable()); 214 } catch (IOException e) { 215 LOG.warn("Failed getting table state", e); 216 } 217 return state != null && state.isDisabledOrDisabling(); 218 } 219 220 /** 221 * Run through referenced servers and save off unknown and the dead. 222 */ 223 private void checkServer(RegionLocations locations) { 224 if (this.services == null) { 225 // Can't do this test if no services. 226 return; 227 } 228 if (locations == null) { 229 return; 230 } 231 if (locations.getRegionLocations() == null) { 232 return; 233 } 234 // Check referenced servers are known/online. Here we are looking 235 // at both the default replica -- the main replica -- and then replica 236 // locations too. 237 for (HRegionLocation location : locations.getRegionLocations()) { 238 if (location == null) { 239 continue; 240 } 241 ServerName sn = location.getServerName(); 242 if (sn == null) { 243 continue; 244 } 245 if (location.getRegion() == null) { 246 LOG.warn("Empty RegionInfo in {}", location); 247 // This should never happen but if it does, will mess up below. 248 continue; 249 } 250 RegionInfo ri = location.getRegion(); 251 // Skip split parent region 252 if (ri.isSplitParent()) { 253 continue; 254 } 255 // skip the offline regions which belong to disabled table. 256 if (isTableDisabled(ri)) { 257 continue; 258 } 259 RegionState rs = this.services.getAssignmentManager().getRegionStates().getRegionState(ri); 260 if (rs == null || rs.isClosedOrAbnormallyClosed()) { 261 // If closed against an 'Unknown Server', that is should be fine. 262 continue; 263 } 264 ServerManager.ServerLiveState state = 265 this.services.getServerManager().isServerKnownAndOnline(sn); 266 switch (state) { 267 case UNKNOWN: 268 this.report.unknownServers.add(new Pair<>(ri, sn)); 269 break; 270 271 default: 272 break; 273 } 274 } 275 } 276 277 /** 278 * @return True iff first row in hbase:meta or if we've broached a new table in hbase:meta 279 */ 280 private boolean isTableTransition(RegionInfo ri) { 281 return this.previous == null || !this.previous.getTable().equals(ri.getTable()); 282 } 283 284 @Override 285 public void close() throws IOException { 286 // This is a table transition... after the last region. Check previous. 287 // Should be last region. If not, its a hole on end of laster table. 288 if (this.previous != null && !this.previous.isLast()) { 289 addHole(this.previous, RegionInfoBuilder.UNDEFINED); 290 } 291 this.closed = true; 292 } 293}