001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master; 019 020import java.io.IOException; 021import java.util.HashMap; 022import java.util.HashSet; 023import java.util.LinkedList; 024import java.util.List; 025import java.util.Map; 026import java.util.Set; 027import java.util.concurrent.locks.ReentrantReadWriteLock; 028 029import org.apache.hadoop.fs.FileSystem; 030import org.apache.hadoop.fs.Path; 031import org.apache.hadoop.hbase.ScheduledChore; 032import org.apache.hadoop.hbase.ServerName; 033import org.apache.hadoop.hbase.client.RegionInfo; 034import org.apache.hadoop.hbase.client.TableState; 035import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 036import org.apache.hadoop.hbase.util.FSUtils; 037import org.apache.hadoop.hbase.util.HbckRegionInfo; 038import org.apache.hadoop.hbase.util.Pair; 039import org.apache.yetus.audience.InterfaceAudience; 040import org.apache.yetus.audience.InterfaceStability; 041import org.slf4j.Logger; 042import org.slf4j.LoggerFactory; 043 044/** 045 * Used to do the hbck checking job at master side. 046 */ 047@InterfaceAudience.Private 048@InterfaceStability.Evolving 049public class HbckChore extends ScheduledChore { 050 private static final Logger LOG = LoggerFactory.getLogger(HbckChore.class.getName()); 051 052 private static final String HBCK_CHORE_INTERVAL = "hbase.master.hbck.chore.interval"; 053 private static final int DEFAULT_HBCK_CHORE_INTERVAL = 60 * 60 * 1000; 054 055 private final MasterServices master; 056 057 /** 058 * This map contains the state of all hbck items. It maps from encoded region 059 * name to HbckRegionInfo structure. The information contained in HbckRegionInfo is used 060 * to detect and correct consistency (hdfs/meta/deployment) problems. 061 */ 062 private final Map<String, HbckRegionInfo> regionInfoMap = new HashMap<>(); 063 064 private final Set<String> disabledTableRegions = new HashSet<>(); 065 private final Set<String> splitParentRegions = new HashSet<>(); 066 067 /** 068 * The regions only opened on RegionServers, but no region info in meta. 069 */ 070 private final Map<String, ServerName> orphanRegionsOnRS = new HashMap<>(); 071 /** 072 * The regions have directory on FileSystem, but no region info in meta. 073 */ 074 private final Map<String, Path> orphanRegionsOnFS = new HashMap<>(); 075 /** 076 * The inconsistent regions. There are three case: 077 * case 1. Master thought this region opened, but no regionserver reported it. 078 * case 2. Master thought this region opened on Server1, but regionserver reported Server2 079 * case 3. More than one regionservers reported opened this region 080 */ 081 private final Map<String, Pair<ServerName, List<ServerName>>> inconsistentRegions = 082 new HashMap<>(); 083 084 /** 085 * The "snapshot" is used to save the last round's HBCK checking report. 086 */ 087 private final Map<String, ServerName> orphanRegionsOnRSSnapshot = new HashMap<>(); 088 private final Map<String, Path> orphanRegionsOnFSSnapshot = new HashMap<>(); 089 private final Map<String, Pair<ServerName, List<ServerName>>> inconsistentRegionsSnapshot = 090 new HashMap<>(); 091 092 /** 093 * The "snapshot" may be changed after checking. And this checking report "snapshot" may be 094 * accessed by web ui. Use this rwLock to synchronize. 095 */ 096 ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); 097 098 /** 099 * When running, the "snapshot" may be changed when this round's checking finish. 100 */ 101 private volatile boolean running = false; 102 private volatile long checkingStartTimestamp = 0; 103 private volatile long checkingEndTimestamp = 0; 104 105 private boolean disabled = false; 106 107 public HbckChore(MasterServices master) { 108 super("HbckChore-", master, 109 master.getConfiguration().getInt(HBCK_CHORE_INTERVAL, DEFAULT_HBCK_CHORE_INTERVAL)); 110 this.master = master; 111 int interval = 112 master.getConfiguration().getInt(HBCK_CHORE_INTERVAL, DEFAULT_HBCK_CHORE_INTERVAL); 113 if (interval <= 0) { 114 LOG.warn(HBCK_CHORE_INTERVAL + " is <=0 hence disabling hbck chore"); 115 disableChore(); 116 } 117 } 118 119 @Override 120 protected synchronized void chore() { 121 if (isDisabled() || isRunning()) { 122 LOG.warn("hbckChore is either disabled or is already running. Can't run the chore"); 123 return; 124 } 125 running = true; 126 regionInfoMap.clear(); 127 disabledTableRegions.clear(); 128 splitParentRegions.clear(); 129 orphanRegionsOnRS.clear(); 130 orphanRegionsOnFS.clear(); 131 inconsistentRegions.clear(); 132 checkingStartTimestamp = EnvironmentEdgeManager.currentTime(); 133 loadRegionsFromInMemoryState(); 134 loadRegionsFromRSReport(); 135 try { 136 loadRegionsFromFS(); 137 } catch (IOException e) { 138 LOG.warn("Failed to load the regions from filesystem", e); 139 } 140 saveCheckResultToSnapshot(); 141 running = false; 142 } 143 144 // This function does the sanity checks of making sure the chore is not run when it is 145 // disabled or when it's already running. It returns whether the chore was actually run or not. 146 protected boolean runChore() { 147 if (isDisabled() || isRunning()) { 148 if (isDisabled()) { 149 LOG.warn("hbck chore is disabled! Set " + HBCK_CHORE_INTERVAL + " > 0 to enable it."); 150 } else { 151 LOG.warn("hbck chore already running. Can't run till it finishes."); 152 } 153 return false; 154 } 155 chore(); 156 return true; 157 } 158 159 private void disableChore() { 160 this.disabled = true; 161 } 162 163 public boolean isDisabled() { 164 return this.disabled; 165 } 166 167 private void saveCheckResultToSnapshot() { 168 // Need synchronized here, as this "snapshot" may be access by web ui. 169 rwLock.writeLock().lock(); 170 try { 171 orphanRegionsOnRSSnapshot.clear(); 172 orphanRegionsOnRS.entrySet() 173 .forEach(e -> orphanRegionsOnRSSnapshot.put(e.getKey(), e.getValue())); 174 orphanRegionsOnFSSnapshot.clear(); 175 orphanRegionsOnFS.entrySet() 176 .forEach(e -> orphanRegionsOnFSSnapshot.put(e.getKey(), e.getValue())); 177 inconsistentRegionsSnapshot.clear(); 178 inconsistentRegions.entrySet() 179 .forEach(e -> inconsistentRegionsSnapshot.put(e.getKey(), e.getValue())); 180 checkingEndTimestamp = EnvironmentEdgeManager.currentTime(); 181 } finally { 182 rwLock.writeLock().unlock(); 183 } 184 } 185 186 private void loadRegionsFromInMemoryState() { 187 List<RegionState> regionStates = 188 master.getAssignmentManager().getRegionStates().getRegionStates(); 189 for (RegionState regionState : regionStates) { 190 RegionInfo regionInfo = regionState.getRegion(); 191 if (master.getTableStateManager() 192 .isTableState(regionInfo.getTable(), TableState.State.DISABLED)) { 193 disabledTableRegions.add(regionInfo.getEncodedName()); 194 } 195 if (regionInfo.isSplitParent()) { 196 splitParentRegions.add(regionInfo.getEncodedName()); 197 } 198 HbckRegionInfo.MetaEntry metaEntry = 199 new HbckRegionInfo.MetaEntry(regionInfo, regionState.getServerName(), 200 regionState.getStamp()); 201 regionInfoMap.put(regionInfo.getEncodedName(), new HbckRegionInfo(metaEntry)); 202 } 203 LOG.info("Loaded {} regions from in-memory state of AssignmentManager", regionStates.size()); 204 } 205 206 private void loadRegionsFromRSReport() { 207 int numRegions = 0; 208 Map<ServerName, Set<byte[]>> rsReports = master.getAssignmentManager().getRSReports(); 209 for (Map.Entry<ServerName, Set<byte[]>> entry : rsReports.entrySet()) { 210 ServerName serverName = entry.getKey(); 211 for (byte[] regionName : entry.getValue()) { 212 String encodedRegionName = RegionInfo.encodeRegionName(regionName); 213 HbckRegionInfo hri = regionInfoMap.get(encodedRegionName); 214 if (hri == null) { 215 orphanRegionsOnRS.put(encodedRegionName, serverName); 216 continue; 217 } 218 hri.addServer(hri.getMetaEntry(), serverName); 219 } 220 numRegions += entry.getValue().size(); 221 } 222 LOG.info("Loaded {} regions from {} regionservers' reports and found {} orphan regions", 223 numRegions, rsReports.size(), orphanRegionsOnFS.size()); 224 225 for (Map.Entry<String, HbckRegionInfo> entry : regionInfoMap.entrySet()) { 226 String encodedRegionName = entry.getKey(); 227 HbckRegionInfo hri = entry.getValue(); 228 ServerName locationInMeta = hri.getMetaEntry().getRegionServer(); 229 if (hri.getDeployedOn().size() == 0) { 230 if (locationInMeta == null) { 231 continue; 232 } 233 // skip the offline region which belong to disabled table. 234 if (disabledTableRegions.contains(encodedRegionName)) { 235 continue; 236 } 237 // skip the split parent regions 238 if (splitParentRegions.contains(encodedRegionName)) { 239 continue; 240 } 241 // Master thought this region opened, but no regionserver reported it. 242 inconsistentRegions.put(encodedRegionName, new Pair<>(locationInMeta, new LinkedList<>())); 243 } else if (hri.getDeployedOn().size() > 1) { 244 // More than one regionserver reported opened this region 245 inconsistentRegions.put(encodedRegionName, new Pair<>(locationInMeta, hri.getDeployedOn())); 246 } else if (!hri.getDeployedOn().get(0).equals(locationInMeta)) { 247 // Master thought this region opened on Server1, but regionserver reported Server2 248 inconsistentRegions.put(encodedRegionName, new Pair<>(locationInMeta, hri.getDeployedOn())); 249 } 250 } 251 } 252 253 private void loadRegionsFromFS() throws IOException { 254 Path rootDir = master.getMasterFileSystem().getRootDir(); 255 FileSystem fs = master.getMasterFileSystem().getFileSystem(); 256 257 int numRegions = 0; 258 List<Path> tableDirs = FSUtils.getTableDirs(fs, rootDir); 259 for (Path tableDir : tableDirs) { 260 List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir); 261 for (Path regionDir : regionDirs) { 262 String encodedRegionName = regionDir.getName(); 263 HbckRegionInfo hri = regionInfoMap.get(encodedRegionName); 264 if (hri == null) { 265 orphanRegionsOnFS.put(encodedRegionName, regionDir); 266 continue; 267 } 268 HbckRegionInfo.HdfsEntry hdfsEntry = new HbckRegionInfo.HdfsEntry(regionDir); 269 hri.setHdfsEntry(hdfsEntry); 270 } 271 numRegions += regionDirs.size(); 272 } 273 LOG.info("Loaded {} tables {} regions from filesyetem and found {} orphan regions", 274 tableDirs.size(), numRegions, orphanRegionsOnFS.size()); 275 } 276 277 /** 278 * When running, the HBCK report may be changed later. 279 */ 280 public boolean isRunning() { 281 return running; 282 } 283 284 /** 285 * @return the regions only opened on RegionServers, but no region info in meta. 286 */ 287 public Map<String, ServerName> getOrphanRegionsOnRS() { 288 // Need synchronized here, as this "snapshot" may be changed after checking. 289 rwLock.readLock().lock(); 290 try { 291 return this.orphanRegionsOnRSSnapshot; 292 } finally { 293 rwLock.readLock().unlock(); 294 } 295 } 296 297 /** 298 * @return the regions have directory on FileSystem, but no region info in meta. 299 */ 300 public Map<String, Path> getOrphanRegionsOnFS() { 301 // Need synchronized here, as this "snapshot" may be changed after checking. 302 rwLock.readLock().lock(); 303 try { 304 return this.orphanRegionsOnFSSnapshot; 305 } finally { 306 rwLock.readLock().unlock(); 307 } 308 } 309 310 /** 311 * Found the inconsistent regions. There are three case: 312 * case 1. Master thought this region opened, but no regionserver reported it. 313 * case 2. Master thought this region opened on Server1, but regionserver reported Server2 314 * case 3. More than one regionservers reported opened this region 315 * 316 * @return the map of inconsistent regions. Key is the region name. Value is a pair of location in 317 * meta and the regionservers which reported opened this region. 318 */ 319 public Map<String, Pair<ServerName, List<ServerName>>> getInconsistentRegions() { 320 // Need synchronized here, as this "snapshot" may be changed after checking. 321 rwLock.readLock().lock(); 322 try { 323 return this.inconsistentRegionsSnapshot; 324 } finally { 325 rwLock.readLock().unlock(); 326 } 327 } 328 329 /** 330 * Used for web ui to show when the HBCK checking started. 331 */ 332 public long getCheckingStartTimestamp() { 333 return this.checkingStartTimestamp; 334 } 335 336 /** 337 * Used for web ui to show when the HBCK checking report generated. 338 */ 339 public long getCheckingEndTimestamp() { 340 return this.checkingEndTimestamp; 341 } 342}