001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.janitor; 019 020import java.io.IOException; 021import java.io.InputStream; 022import java.util.Comparator; 023import java.util.HashSet; 024import java.util.List; 025import java.util.Map; 026import java.util.Properties; 027import java.util.concurrent.atomic.AtomicBoolean; 028import java.util.stream.Collectors; 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.FileSystem; 031import org.apache.hadoop.fs.Path; 032import org.apache.hadoop.hbase.HBaseConfiguration; 033import org.apache.hadoop.hbase.HConstants; 034import org.apache.hadoop.hbase.MetaTableAccessor; 035import org.apache.hadoop.hbase.ScheduledChore; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 038import org.apache.hadoop.hbase.client.Connection; 039import org.apache.hadoop.hbase.client.ConnectionFactory; 040import org.apache.hadoop.hbase.client.Get; 041import org.apache.hadoop.hbase.client.Put; 042import org.apache.hadoop.hbase.client.RegionInfo; 043import org.apache.hadoop.hbase.client.Result; 044import org.apache.hadoop.hbase.client.Table; 045import org.apache.hadoop.hbase.client.TableDescriptor; 046import org.apache.hadoop.hbase.master.MasterServices; 047import org.apache.hadoop.hbase.master.assignment.AssignmentManager; 048import org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure; 049import org.apache.hadoop.hbase.master.assignment.GCRegionProcedure; 050import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 051import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 052import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 053import org.apache.hadoop.hbase.util.Bytes; 054import org.apache.hadoop.hbase.util.CommonFSUtils; 055import org.apache.hadoop.hbase.util.Pair; 056import org.apache.hadoop.hbase.util.PairOfSameType; 057import org.apache.hadoop.hbase.util.Threads; 058import org.apache.yetus.audience.InterfaceAudience; 059import org.slf4j.Logger; 060import org.slf4j.LoggerFactory; 061 062/** 063 * A janitor for the catalog tables. Scans the <code>hbase:meta</code> catalog table on a period. 064 * Makes a lastReport on state of hbase:meta. Looks for unused regions to garbage collect. Scan of 065 * hbase:meta runs if we are NOT in maintenance mode, if we are NOT shutting down, AND if the 066 * assignmentmanager is loaded. Playing it safe, we will garbage collect no-longer needed region 067 * references only if there are no regions-in-transition (RIT). 068 */ 069// TODO: Only works with single hbase:meta region currently. Fix. 070// TODO: Should it start over every time? Could it continue if runs into problem? Only if 071// problem does not mess up 'results'. 072// TODO: Do more by way of 'repair'; see note on unknownServers below. 073@InterfaceAudience.Private 074public class CatalogJanitor extends ScheduledChore { 075 076 public static final int DEFAULT_HBASE_CATALOGJANITOR_INTERVAL = 300 * 1000; 077 078 private static final Logger LOG = LoggerFactory.getLogger(CatalogJanitor.class.getName()); 079 080 private final AtomicBoolean alreadyRunning = new AtomicBoolean(false); 081 private final AtomicBoolean enabled = new AtomicBoolean(true); 082 private final MasterServices services; 083 084 /** 085 * Saved report from last hbase:meta scan to completion. May be stale if having trouble completing 086 * scan. Check its date. 087 */ 088 private volatile CatalogJanitorReport lastReport; 089 090 public CatalogJanitor(final MasterServices services) { 091 super("CatalogJanitor-" + services.getServerName().toShortString(), services, 092 services.getConfiguration().getInt("hbase.catalogjanitor.interval", 093 DEFAULT_HBASE_CATALOGJANITOR_INTERVAL)); 094 this.services = services; 095 } 096 097 @Override 098 protected boolean initialChore() { 099 try { 100 if (getEnabled()) { 101 scan(); 102 } 103 } catch (IOException e) { 104 LOG.warn("Failed initial janitorial scan of hbase:meta table", e); 105 return false; 106 } 107 return true; 108 } 109 110 public boolean setEnabled(final boolean enabled) { 111 boolean alreadyEnabled = this.enabled.getAndSet(enabled); 112 // If disabling is requested on an already enabled chore, we could have an active 113 // scan still going on, callers might not be aware of that and do further action thinkng 114 // that no action would be from this chore. In this case, the right action is to wait for 115 // the active scan to complete before exiting this function. 116 if (!enabled && alreadyEnabled) { 117 while (alreadyRunning.get()) { 118 Threads.sleepWithoutInterrupt(100); 119 } 120 } 121 return alreadyEnabled; 122 } 123 124 public boolean getEnabled() { 125 return this.enabled.get(); 126 } 127 128 @Override 129 protected void chore() { 130 try { 131 AssignmentManager am = this.services.getAssignmentManager(); 132 if ( 133 getEnabled() && !this.services.isInMaintenanceMode() 134 && !this.services.getServerManager().isClusterShutdown() && isMetaLoaded(am) 135 ) { 136 scan(); 137 } else { 138 LOG.warn("CatalogJanitor is disabled! Enabled=" + getEnabled() + ", maintenanceMode=" 139 + this.services.isInMaintenanceMode() + ", am=" + am + ", metaLoaded=" + isMetaLoaded(am) 140 + ", hasRIT=" + isRIT(am) + " clusterShutDown=" 141 + this.services.getServerManager().isClusterShutdown()); 142 } 143 } catch (IOException e) { 144 LOG.warn("Failed janitorial scan of hbase:meta table", e); 145 } 146 } 147 148 private static boolean isMetaLoaded(AssignmentManager am) { 149 return am != null && am.isMetaLoaded(); 150 } 151 152 private static boolean isRIT(AssignmentManager am) { 153 return isMetaLoaded(am) && am.hasRegionsInTransition(); 154 } 155 156 /** 157 * Run janitorial scan of catalog <code>hbase:meta</code> table looking for garbage to collect. 158 * @return How many items gc'd whether for merge or split. Returns -1 if previous scan is in 159 * progress. 160 */ 161 public int scan() throws IOException { 162 int gcs = 0; 163 try { 164 if (!alreadyRunning.compareAndSet(false, true)) { 165 if (LOG.isDebugEnabled()) { 166 LOG.debug("CatalogJanitor already running"); 167 } 168 // -1 indicates previous scan is in progress 169 return -1; 170 } 171 this.lastReport = scanForReport(); 172 if (!this.lastReport.isEmpty()) { 173 LOG.warn(this.lastReport.toString()); 174 } else { 175 if (LOG.isDebugEnabled()) { 176 LOG.debug(this.lastReport.toString()); 177 } 178 } 179 180 updateAssignmentManagerMetrics(); 181 182 Map<RegionInfo, Result> mergedRegions = this.lastReport.mergedRegions; 183 for (Map.Entry<RegionInfo, Result> e : mergedRegions.entrySet()) { 184 if (this.services.isInMaintenanceMode()) { 185 // Stop cleaning if the master is in maintenance mode 186 LOG.debug("In maintenence mode, not cleaning"); 187 break; 188 } 189 190 List<RegionInfo> parents = MetaTableAccessor.getMergeRegions(e.getValue().rawCells()); 191 if (parents != null && cleanMergeRegion(e.getKey(), parents)) { 192 gcs++; 193 } 194 } 195 // Clean split parents 196 Map<RegionInfo, Result> splitParents = this.lastReport.splitParents; 197 198 // Now work on our list of found parents. See if any we can clean up. 199 HashSet<String> parentNotCleaned = new HashSet<>(); 200 for (Map.Entry<RegionInfo, Result> e : splitParents.entrySet()) { 201 if (this.services.isInMaintenanceMode()) { 202 // Stop cleaning if the master is in maintenance mode 203 if (LOG.isDebugEnabled()) { 204 LOG.debug("In maintenence mode, not cleaning"); 205 } 206 break; 207 } 208 209 if ( 210 !parentNotCleaned.contains(e.getKey().getEncodedName()) 211 && cleanParent(e.getKey(), e.getValue()) 212 ) { 213 gcs++; 214 } else { 215 // We could not clean the parent, so it's daughters should not be 216 // cleaned either (HBASE-6160) 217 PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(e.getValue()); 218 parentNotCleaned.add(daughters.getFirst().getEncodedName()); 219 parentNotCleaned.add(daughters.getSecond().getEncodedName()); 220 } 221 } 222 return gcs; 223 } finally { 224 alreadyRunning.set(false); 225 } 226 } 227 228 /** 229 * Scan hbase:meta. 230 * @return Return generated {@link CatalogJanitorReport} 231 */ 232 // will be override in tests. 233 protected CatalogJanitorReport scanForReport() throws IOException { 234 ReportMakingVisitor visitor = new ReportMakingVisitor(this.services); 235 // Null tablename means scan all of meta. 236 MetaTableAccessor.scanMetaForTableRegions(this.services.getConnection(), visitor, null); 237 return visitor.getReport(); 238 } 239 240 /** Returns Returns last published Report that comes of last successful scan of hbase:meta. */ 241 public CatalogJanitorReport getLastReport() { 242 return this.lastReport; 243 } 244 245 /** 246 * If merged region no longer holds reference to the merge regions, archive merge region on hdfs 247 * and perform deleting references in hbase:meta 248 * @return true if we delete references in merged region on hbase:meta and archive the files on 249 * the file system 250 */ 251 private boolean cleanMergeRegion(final RegionInfo mergedRegion, List<RegionInfo> parents) 252 throws IOException { 253 if (LOG.isDebugEnabled()) { 254 LOG.debug("Cleaning merged region {}", mergedRegion); 255 } 256 FileSystem fs = this.services.getMasterFileSystem().getFileSystem(); 257 Path rootdir = this.services.getMasterFileSystem().getRootDir(); 258 Path tabledir = CommonFSUtils.getTableDir(rootdir, mergedRegion.getTable()); 259 TableDescriptor htd = getDescriptor(mergedRegion.getTable()); 260 HRegionFileSystem regionFs = null; 261 try { 262 regionFs = HRegionFileSystem.openRegionFromFileSystem(this.services.getConfiguration(), fs, 263 tabledir, mergedRegion, true); 264 } catch (IOException e) { 265 LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName()); 266 } 267 if (regionFs == null || !regionFs.hasReferences(htd)) { 268 if (LOG.isDebugEnabled()) { 269 LOG.debug( 270 "Deleting parents ({}) from fs; merged child {} no longer holds references", parents 271 .stream().map(r -> RegionInfo.getShortNameToLog(r)).collect(Collectors.joining(", ")), 272 mergedRegion); 273 } 274 ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor(); 275 GCMultipleMergedRegionsProcedure mergeRegionProcedure = 276 new GCMultipleMergedRegionsProcedure(pe.getEnvironment(), mergedRegion, parents); 277 pe.submitProcedure(mergeRegionProcedure); 278 if (LOG.isDebugEnabled()) { 279 LOG.debug("Submitted procedure {} for merged region {}", mergeRegionProcedure, 280 mergedRegion); 281 } 282 return true; 283 } 284 return false; 285 } 286 287 /** 288 * Compare HRegionInfos in a way that has split parents sort BEFORE their daughters. 289 */ 290 static class SplitParentFirstComparator implements Comparator<RegionInfo> { 291 Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator(); 292 293 @Override 294 public int compare(RegionInfo left, RegionInfo right) { 295 // This comparator differs from the one RegionInfo in that it sorts 296 // parent before daughters. 297 if (left == null) { 298 return -1; 299 } 300 if (right == null) { 301 return 1; 302 } 303 // Same table name. 304 int result = left.getTable().compareTo(right.getTable()); 305 if (result != 0) { 306 return result; 307 } 308 // Compare start keys. 309 result = Bytes.compareTo(left.getStartKey(), right.getStartKey()); 310 if (result != 0) { 311 return result; 312 } 313 // Compare end keys, but flip the operands so parent comes first 314 result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey()); 315 316 return result; 317 } 318 } 319 320 static boolean cleanParent(MasterServices services, RegionInfo parent, Result rowContent) 321 throws IOException { 322 if (LOG.isDebugEnabled()) { 323 LOG.debug("Cleaning parent region {}", parent); 324 } 325 // Check whether it is a merged region and if it is clean of references. 326 if (MetaTableAccessor.hasMergeRegions(rowContent.rawCells())) { 327 // Wait until clean of merge parent regions first 328 if (LOG.isDebugEnabled()) { 329 LOG.debug("Region {} has merge parents, cleaning them first", parent); 330 } 331 return false; 332 } 333 // Run checks on each daughter split. 334 PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent); 335 Pair<Boolean, Boolean> a = checkDaughterInFs(services, parent, daughters.getFirst()); 336 Pair<Boolean, Boolean> b = checkDaughterInFs(services, parent, daughters.getSecond()); 337 if (hasNoReferences(a) && hasNoReferences(b)) { 338 String daughterA = 339 daughters.getFirst() != null ? daughters.getFirst().getShortNameToLog() : "null"; 340 String daughterB = 341 daughters.getSecond() != null ? daughters.getSecond().getShortNameToLog() : "null"; 342 if (LOG.isDebugEnabled()) { 343 LOG.debug("Deleting region " + parent.getShortNameToLog() + " because daughters -- " 344 + daughterA + ", " + daughterB + " -- no longer hold references"); 345 } 346 ProcedureExecutor<MasterProcedureEnv> pe = services.getMasterProcedureExecutor(); 347 GCRegionProcedure gcRegionProcedure = new GCRegionProcedure(pe.getEnvironment(), parent); 348 pe.submitProcedure(gcRegionProcedure); 349 if (LOG.isDebugEnabled()) { 350 LOG.debug("Submitted procedure {} for split parent {}", gcRegionProcedure, parent); 351 } 352 return true; 353 } else { 354 if (LOG.isDebugEnabled()) { 355 if (!hasNoReferences(a)) { 356 LOG.debug("Deferring removal of region {} because daughter {} still has references", 357 parent, daughters.getFirst()); 358 } 359 if (!hasNoReferences(b)) { 360 LOG.debug("Deferring removal of region {} because daughter {} still has references", 361 parent, daughters.getSecond()); 362 } 363 } 364 } 365 return false; 366 } 367 368 /** 369 * If daughters no longer hold reference to the parents, delete the parent. 370 * @param parent RegionInfo of split offlined parent 371 * @param rowContent Content of <code>parent</code> row in <code>metaRegionName</code> 372 * @return True if we removed <code>parent</code> from meta table and from the filesystem. 373 */ 374 private boolean cleanParent(final RegionInfo parent, Result rowContent) throws IOException { 375 return cleanParent(services, parent, rowContent); 376 } 377 378 /** 379 * @param p A pair where the first boolean says whether or not the daughter region directory 380 * exists in the filesystem and then the second boolean says whether the daughter has 381 * references to the parent. 382 * @return True the passed <code>p</code> signifies no references. 383 */ 384 private static boolean hasNoReferences(final Pair<Boolean, Boolean> p) { 385 return !p.getFirst() || !p.getSecond(); 386 } 387 388 /** 389 * Checks if a daughter region -- either splitA or splitB -- still holds references to parent. 390 * @param parent Parent region 391 * @param daughter Daughter region 392 * @return A pair where the first boolean says whether or not the daughter region directory exists 393 * in the filesystem and then the second boolean says whether the daughter has references 394 * to the parent. 395 */ 396 private static Pair<Boolean, Boolean> checkDaughterInFs(MasterServices services, 397 final RegionInfo parent, final RegionInfo daughter) throws IOException { 398 if (daughter == null) { 399 return new Pair<>(Boolean.FALSE, Boolean.FALSE); 400 } 401 402 FileSystem fs = services.getMasterFileSystem().getFileSystem(); 403 Path rootdir = services.getMasterFileSystem().getRootDir(); 404 Path tabledir = CommonFSUtils.getTableDir(rootdir, daughter.getTable()); 405 406 Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName()); 407 408 HRegionFileSystem regionFs; 409 410 try { 411 if (!CommonFSUtils.isExists(fs, daughterRegionDir)) { 412 return new Pair<>(Boolean.FALSE, Boolean.FALSE); 413 } 414 } catch (IOException ioe) { 415 LOG.error("Error trying to determine if daughter region exists, " 416 + "assuming exists and has references", ioe); 417 return new Pair<>(Boolean.TRUE, Boolean.TRUE); 418 } 419 420 boolean references = false; 421 TableDescriptor parentDescriptor = services.getTableDescriptors().get(parent.getTable()); 422 try { 423 regionFs = HRegionFileSystem.openRegionFromFileSystem(services.getConfiguration(), fs, 424 tabledir, daughter, true); 425 426 for (ColumnFamilyDescriptor family : parentDescriptor.getColumnFamilies()) { 427 references = regionFs.hasReferences(family.getNameAsString()); 428 if (references) { 429 break; 430 } 431 } 432 } catch (IOException e) { 433 LOG.error("Error trying to determine referenced files from : " + daughter.getEncodedName() 434 + ", to: " + parent.getEncodedName() + " assuming has references", e); 435 return new Pair<>(Boolean.TRUE, Boolean.TRUE); 436 } 437 return new Pair<>(Boolean.TRUE, references); 438 } 439 440 private TableDescriptor getDescriptor(final TableName tableName) throws IOException { 441 return this.services.getTableDescriptors().get(tableName); 442 } 443 444 private void updateAssignmentManagerMetrics() { 445 services.getAssignmentManager().getAssignmentManagerMetrics() 446 .updateHoles(lastReport.getHoles().size()); 447 services.getAssignmentManager().getAssignmentManagerMetrics() 448 .updateOverlaps(lastReport.getOverlaps().size()); 449 services.getAssignmentManager().getAssignmentManagerMetrics() 450 .updateUnknownServerRegions(lastReport.getUnknownServers().size()); 451 services.getAssignmentManager().getAssignmentManagerMetrics() 452 .updateEmptyRegionInfoRegions(lastReport.getEmptyRegionInfo().size()); 453 } 454 455 private static void checkLog4jProperties() { 456 String filename = "log4j.properties"; 457 try (final InputStream inStream = 458 CatalogJanitor.class.getClassLoader().getResourceAsStream(filename)) { 459 if (inStream != null) { 460 new Properties().load(inStream); 461 } else { 462 System.out.println("No " + filename + " on classpath; Add one else no logging output!"); 463 } 464 } catch (IOException e) { 465 LOG.error("Log4j check failed", e); 466 } 467 } 468 469 /** 470 * For testing against a cluster. Doesn't have a MasterServices context so does not report on good 471 * vs bad servers. 472 */ 473 public static void main(String[] args) throws IOException { 474 checkLog4jProperties(); 475 ReportMakingVisitor visitor = new ReportMakingVisitor(null); 476 Configuration configuration = HBaseConfiguration.create(); 477 configuration.setBoolean("hbase.defaults.for.version.skip", true); 478 try (Connection connection = ConnectionFactory.createConnection(configuration)) { 479 /* 480 * Used to generate an overlap. 481 */ 482 Get g = new Get(Bytes.toBytes("t2,40,1564119846424.1db8c57d64e0733e0f027aaeae7a0bf0.")); 483 g.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); 484 try (Table t = connection.getTable(TableName.META_TABLE_NAME)) { 485 Result r = t.get(g); 486 byte[] row = g.getRow(); 487 row[row.length - 2] <<= row[row.length - 2]; 488 Put p = new Put(g.getRow()); 489 p.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, 490 r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER)); 491 t.put(p); 492 } 493 MetaTableAccessor.scanMetaForTableRegions(connection, visitor, null); 494 CatalogJanitorReport report = visitor.getReport(); 495 LOG.info(report != null ? report.toString() : "empty"); 496 } 497 } 498}