001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.janitor; 019 020import java.io.IOException; 021import java.io.InputStream; 022import java.util.Comparator; 023import java.util.HashSet; 024import java.util.List; 025import java.util.Map; 026import java.util.Properties; 027import java.util.concurrent.atomic.AtomicBoolean; 028import java.util.stream.Collectors; 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.FileSystem; 031import org.apache.hadoop.fs.Path; 032import org.apache.hadoop.hbase.HBaseConfiguration; 033import org.apache.hadoop.hbase.HConstants; 034import org.apache.hadoop.hbase.MetaTableAccessor; 035import org.apache.hadoop.hbase.ScheduledChore; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.client.Connection; 038import org.apache.hadoop.hbase.client.ConnectionFactory; 039import org.apache.hadoop.hbase.client.Get; 040import org.apache.hadoop.hbase.client.Put; 041import org.apache.hadoop.hbase.client.RegionInfo; 042import org.apache.hadoop.hbase.client.Result; 043import org.apache.hadoop.hbase.client.Table; 044import org.apache.hadoop.hbase.client.TableDescriptor; 045import org.apache.hadoop.hbase.master.MasterServices; 046import org.apache.hadoop.hbase.master.assignment.AssignmentManager; 047import org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure; 048import org.apache.hadoop.hbase.master.assignment.GCRegionProcedure; 049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 050import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 051import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 052import org.apache.hadoop.hbase.util.Bytes; 053import org.apache.hadoop.hbase.util.CommonFSUtils; 054import org.apache.hadoop.hbase.util.Pair; 055import org.apache.hadoop.hbase.util.PairOfSameType; 056import org.apache.hadoop.hbase.util.Threads; 057import org.apache.yetus.audience.InterfaceAudience; 058import org.slf4j.Logger; 059import org.slf4j.LoggerFactory; 060 061/** 062 * A janitor for the catalog tables. Scans the <code>hbase:meta</code> catalog table on a period. 063 * Makes a lastReport on state of hbase:meta. Looks for unused regions to garbage collect. Scan of 064 * hbase:meta runs if we are NOT in maintenance mode, if we are NOT shutting down, AND if the 065 * assignmentmanager is loaded. Playing it safe, we will garbage collect no-longer needed region 066 * references only if there are no regions-in-transition (RIT). 067 */ 068// TODO: Only works with single hbase:meta region currently. Fix. 069// TODO: Should it start over every time? Could it continue if runs into problem? Only if 070// problem does not mess up 'results'. 071// TODO: Do more by way of 'repair'; see note on unknownServers below. 072@InterfaceAudience.Private 073public class CatalogJanitor extends ScheduledChore { 074 075 public static final int DEFAULT_HBASE_CATALOGJANITOR_INTERVAL = 300 * 1000; 076 077 private static final Logger LOG = LoggerFactory.getLogger(CatalogJanitor.class.getName()); 078 079 private final AtomicBoolean alreadyRunning = new AtomicBoolean(false); 080 private final AtomicBoolean enabled = new AtomicBoolean(true); 081 private final MasterServices services; 082 083 /** 084 * Saved report from last hbase:meta scan to completion. May be stale if having trouble completing 085 * scan. Check its date. 086 */ 087 private volatile CatalogJanitorReport lastReport; 088 089 public CatalogJanitor(final MasterServices services) { 090 super("CatalogJanitor-" + services.getServerName().toShortString(), services, 091 services.getConfiguration().getInt("hbase.catalogjanitor.interval", 092 DEFAULT_HBASE_CATALOGJANITOR_INTERVAL)); 093 this.services = services; 094 } 095 096 @Override 097 protected boolean initialChore() { 098 try { 099 if (getEnabled()) { 100 scan(); 101 } 102 } catch (IOException e) { 103 LOG.warn("Failed initial janitorial scan of hbase:meta table", e); 104 return false; 105 } 106 return true; 107 } 108 109 public boolean setEnabled(final boolean enabled) { 110 boolean alreadyEnabled = this.enabled.getAndSet(enabled); 111 // If disabling is requested on an already enabled chore, we could have an active 112 // scan still going on, callers might not be aware of that and do further action thinkng 113 // that no action would be from this chore. In this case, the right action is to wait for 114 // the active scan to complete before exiting this function. 115 if (!enabled && alreadyEnabled) { 116 while (alreadyRunning.get()) { 117 Threads.sleepWithoutInterrupt(100); 118 } 119 } 120 return alreadyEnabled; 121 } 122 123 public boolean getEnabled() { 124 return this.enabled.get(); 125 } 126 127 @Override 128 protected void chore() { 129 try { 130 AssignmentManager am = this.services.getAssignmentManager(); 131 if ( 132 getEnabled() && !this.services.isInMaintenanceMode() 133 && !this.services.getServerManager().isClusterShutdown() && isMetaLoaded(am) 134 ) { 135 scan(); 136 } else { 137 LOG.warn("CatalogJanitor is disabled! Enabled=" + getEnabled() + ", maintenanceMode=" 138 + this.services.isInMaintenanceMode() + ", am=" + am + ", metaLoaded=" + isMetaLoaded(am) 139 + ", hasRIT=" + isRIT(am) + " clusterShutDown=" 140 + this.services.getServerManager().isClusterShutdown()); 141 } 142 } catch (IOException e) { 143 LOG.warn("Failed janitorial scan of hbase:meta table", e); 144 } 145 } 146 147 private static boolean isMetaLoaded(AssignmentManager am) { 148 return am != null && am.isMetaLoaded(); 149 } 150 151 private static boolean isRIT(AssignmentManager am) { 152 return isMetaLoaded(am) && am.hasRegionsInTransition(); 153 } 154 155 /** 156 * Run janitorial scan of catalog <code>hbase:meta</code> table looking for garbage to collect. 157 * @return How many items gc'd whether for merge or split. Returns -1 if previous scan is in 158 * progress. 159 */ 160 public int scan() throws IOException { 161 int gcs = 0; 162 try { 163 if (!alreadyRunning.compareAndSet(false, true)) { 164 if (LOG.isDebugEnabled()) { 165 LOG.debug("CatalogJanitor already running"); 166 } 167 // -1 indicates previous scan is in progress 168 return -1; 169 } 170 this.lastReport = scanForReport(); 171 if (!this.lastReport.isEmpty()) { 172 LOG.warn(this.lastReport.toString()); 173 } else { 174 if (LOG.isDebugEnabled()) { 175 LOG.debug(this.lastReport.toString()); 176 } 177 } 178 179 updateAssignmentManagerMetrics(); 180 181 Map<RegionInfo, Result> mergedRegions = this.lastReport.mergedRegions; 182 for (Map.Entry<RegionInfo, Result> e : mergedRegions.entrySet()) { 183 if (this.services.isInMaintenanceMode()) { 184 // Stop cleaning if the master is in maintenance mode 185 LOG.debug("In maintenance mode, not cleaning"); 186 break; 187 } 188 189 List<RegionInfo> parents = MetaTableAccessor.getMergeRegions(e.getValue().rawCells()); 190 if (parents != null && cleanMergeRegion(this.services, e.getKey(), parents)) { 191 gcs++; 192 } 193 } 194 // Clean split parents 195 Map<RegionInfo, Result> splitParents = this.lastReport.splitParents; 196 197 // Now work on our list of found parents. See if any we can clean up. 198 HashSet<String> parentNotCleaned = new HashSet<>(); 199 for (Map.Entry<RegionInfo, Result> e : splitParents.entrySet()) { 200 if (this.services.isInMaintenanceMode()) { 201 // Stop cleaning if the master is in maintenance mode 202 if (LOG.isDebugEnabled()) { 203 LOG.debug("In maintenance mode, not cleaning"); 204 } 205 break; 206 } 207 208 if ( 209 !parentNotCleaned.contains(e.getKey().getEncodedName()) 210 && cleanParent(e.getKey(), e.getValue()) 211 ) { 212 gcs++; 213 } else { 214 // We could not clean the parent, so it's daughters should not be 215 // cleaned either (HBASE-6160) 216 PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(e.getValue()); 217 parentNotCleaned.add(daughters.getFirst().getEncodedName()); 218 parentNotCleaned.add(daughters.getSecond().getEncodedName()); 219 } 220 } 221 return gcs; 222 } finally { 223 alreadyRunning.set(false); 224 } 225 } 226 227 /** 228 * Scan hbase:meta. 229 * @return Return generated {@link CatalogJanitorReport} 230 */ 231 // will be override in tests. 232 protected CatalogJanitorReport scanForReport() throws IOException { 233 ReportMakingVisitor visitor = new ReportMakingVisitor(this.services); 234 // Null tablename means scan all of meta. 235 MetaTableAccessor.scanMetaForTableRegions(this.services.getConnection(), visitor, null); 236 return visitor.getReport(); 237 } 238 239 /** Returns Returns last published Report that comes of last successful scan of hbase:meta. */ 240 public CatalogJanitorReport getLastReport() { 241 return this.lastReport; 242 } 243 244 /** 245 * If merged region no longer holds reference to the merge regions, archive merge region on hdfs 246 * and perform deleting references in hbase:meta 247 * @return true if we delete references in merged region on hbase:meta and archive the files on 248 * the file system 249 */ 250 static boolean cleanMergeRegion(MasterServices services, final RegionInfo mergedRegion, 251 List<RegionInfo> parents) throws IOException { 252 if (LOG.isDebugEnabled()) { 253 LOG.debug("Cleaning merged region {}", mergedRegion); 254 } 255 256 Pair<Boolean, Boolean> result = 257 checkRegionReferences(services, mergedRegion.getTable(), mergedRegion); 258 259 if (hasNoReferences(result)) { 260 if (LOG.isDebugEnabled()) { 261 LOG.debug( 262 "Deleting parents ({}) from fs; merged child {} no longer holds references", parents 263 .stream().map(r -> RegionInfo.getShortNameToLog(r)).collect(Collectors.joining(", ")), 264 mergedRegion); 265 } 266 267 ProcedureExecutor<MasterProcedureEnv> pe = services.getMasterProcedureExecutor(); 268 GCMultipleMergedRegionsProcedure mergeRegionProcedure = 269 new GCMultipleMergedRegionsProcedure(pe.getEnvironment(), mergedRegion, parents); 270 pe.submitProcedure(mergeRegionProcedure); 271 if (LOG.isDebugEnabled()) { 272 LOG.debug("Submitted procedure {} for merged region {}", mergeRegionProcedure, 273 mergedRegion); 274 } 275 return true; 276 } else { 277 if (LOG.isDebugEnabled()) { 278 LOG.debug( 279 "Deferring cleanup up of {} parents of merged region {}, because references " 280 + "still exist in merged region or we encountered an exception in checking", 281 parents.size(), mergedRegion.getEncodedName()); 282 } 283 } 284 285 return false; 286 } 287 288 /** 289 * Compare HRegionInfos in a way that has split parents sort BEFORE their daughters. 290 */ 291 static class SplitParentFirstComparator implements Comparator<RegionInfo> { 292 Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator(); 293 294 @Override 295 public int compare(RegionInfo left, RegionInfo right) { 296 // This comparator differs from the one RegionInfo in that it sorts 297 // parent before daughters. 298 if (left == null) { 299 return -1; 300 } 301 if (right == null) { 302 return 1; 303 } 304 // Same table name. 305 int result = left.getTable().compareTo(right.getTable()); 306 if (result != 0) { 307 return result; 308 } 309 // Compare start keys. 310 result = Bytes.compareTo(left.getStartKey(), right.getStartKey()); 311 if (result != 0) { 312 return result; 313 } 314 // Compare end keys, but flip the operands so parent comes first 315 result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey()); 316 317 return result; 318 } 319 } 320 321 static boolean cleanParent(MasterServices services, RegionInfo parent, Result rowContent) 322 throws IOException { 323 if (LOG.isDebugEnabled()) { 324 LOG.debug("Cleaning parent region {}", parent); 325 } 326 // Check whether it is a merged region and if it is clean of references. 327 if (MetaTableAccessor.hasMergeRegions(rowContent.rawCells())) { 328 // Wait until clean of merge parent regions first 329 if (LOG.isDebugEnabled()) { 330 LOG.debug("Region {} has merge parents, cleaning them first", parent); 331 } 332 return false; 333 } 334 // Run checks on each daughter split. 335 PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent); 336 Pair<Boolean, Boolean> a = 337 checkRegionReferences(services, parent.getTable(), daughters.getFirst()); 338 Pair<Boolean, Boolean> b = 339 checkRegionReferences(services, parent.getTable(), daughters.getSecond()); 340 if (hasNoReferences(a) && hasNoReferences(b)) { 341 String daughterA = 342 daughters.getFirst() != null ? daughters.getFirst().getShortNameToLog() : "null"; 343 String daughterB = 344 daughters.getSecond() != null ? daughters.getSecond().getShortNameToLog() : "null"; 345 if (LOG.isDebugEnabled()) { 346 LOG.debug("Deleting region " + parent.getShortNameToLog() + " because daughters -- " 347 + daughterA + ", " + daughterB + " -- no longer hold references"); 348 } 349 ProcedureExecutor<MasterProcedureEnv> pe = services.getMasterProcedureExecutor(); 350 GCRegionProcedure gcRegionProcedure = new GCRegionProcedure(pe.getEnvironment(), parent); 351 pe.submitProcedure(gcRegionProcedure); 352 if (LOG.isDebugEnabled()) { 353 LOG.debug("Submitted procedure {} for split parent {}", gcRegionProcedure, parent); 354 } 355 return true; 356 } else { 357 if (LOG.isDebugEnabled()) { 358 if (!hasNoReferences(a)) { 359 LOG.debug("Deferring removal of region {} because daughter {} still has references", 360 parent, daughters.getFirst()); 361 } 362 if (!hasNoReferences(b)) { 363 LOG.debug("Deferring removal of region {} because daughter {} still has references", 364 parent, daughters.getSecond()); 365 } 366 } 367 } 368 return false; 369 } 370 371 /** 372 * If daughters no longer hold reference to the parents, delete the parent. 373 * @param parent RegionInfo of split offlined parent 374 * @param rowContent Content of <code>parent</code> row in <code>metaRegionName</code> 375 * @return True if we removed <code>parent</code> from meta table and from the filesystem. 376 */ 377 private boolean cleanParent(final RegionInfo parent, Result rowContent) throws IOException { 378 return cleanParent(services, parent, rowContent); 379 } 380 381 /** 382 * @param p A pair where the first boolean says whether or not the daughter region directory 383 * exists in the filesystem and then the second boolean says whether the daughter has 384 * references to the parent. 385 * @return True the passed <code>p</code> signifies no references. 386 */ 387 private static boolean hasNoReferences(final Pair<Boolean, Boolean> p) { 388 return !p.getFirst() || !p.getSecond(); 389 } 390 391 /** 392 * Checks if a region still holds references to parent. 393 * @param tableName The table for the region 394 * @param region The region to check 395 * @return A pair where the first boolean says whether the region directory exists in the 396 * filesystem and then the second boolean says whether the region has references to a 397 * parent. 398 */ 399 private static Pair<Boolean, Boolean> checkRegionReferences(MasterServices services, 400 TableName tableName, RegionInfo region) throws IOException { 401 if (region == null) { 402 return new Pair<>(Boolean.FALSE, Boolean.FALSE); 403 } 404 405 FileSystem fs = services.getMasterFileSystem().getFileSystem(); 406 Path rootdir = services.getMasterFileSystem().getRootDir(); 407 Path tabledir = CommonFSUtils.getTableDir(rootdir, tableName); 408 Path regionDir = new Path(tabledir, region.getEncodedName()); 409 410 try { 411 if (!CommonFSUtils.isExists(fs, regionDir)) { 412 return new Pair<>(Boolean.FALSE, Boolean.FALSE); 413 } 414 } catch (IOException ioe) { 415 LOG.error("Error trying to determine if region exists, assuming exists and has references", 416 ioe); 417 return new Pair<>(Boolean.TRUE, Boolean.TRUE); 418 } 419 420 TableDescriptor tableDescriptor = services.getTableDescriptors().get(tableName); 421 try { 422 HRegionFileSystem regionFs = HRegionFileSystem 423 .openRegionFromFileSystem(services.getConfiguration(), fs, tabledir, region, true); 424 boolean references = regionFs.hasReferences(tableDescriptor); 425 return new Pair<>(Boolean.TRUE, references); 426 } catch (IOException e) { 427 LOG.error("Error trying to determine if region {} has references, assuming it does", 428 region.getEncodedName(), e); 429 return new Pair<>(Boolean.TRUE, Boolean.TRUE); 430 } 431 } 432 433 private void updateAssignmentManagerMetrics() { 434 services.getAssignmentManager().getAssignmentManagerMetrics() 435 .updateHoles(lastReport.getHoles().size()); 436 services.getAssignmentManager().getAssignmentManagerMetrics() 437 .updateOverlaps(lastReport.getOverlaps().size()); 438 services.getAssignmentManager().getAssignmentManagerMetrics() 439 .updateUnknownServerRegions(lastReport.getUnknownServers().size()); 440 services.getAssignmentManager().getAssignmentManagerMetrics() 441 .updateEmptyRegionInfoRegions(lastReport.getEmptyRegionInfo().size()); 442 } 443 444 private static void checkLog4jProperties() { 445 String filename = "log4j.properties"; 446 try (final InputStream inStream = 447 CatalogJanitor.class.getClassLoader().getResourceAsStream(filename)) { 448 if (inStream != null) { 449 new Properties().load(inStream); 450 } else { 451 System.out.println("No " + filename + " on classpath; Add one else no logging output!"); 452 } 453 } catch (IOException e) { 454 LOG.error("Log4j check failed", e); 455 } 456 } 457 458 /** 459 * For testing against a cluster. Doesn't have a MasterServices context so does not report on good 460 * vs bad servers. 461 */ 462 public static void main(String[] args) throws IOException { 463 checkLog4jProperties(); 464 ReportMakingVisitor visitor = new ReportMakingVisitor(null); 465 Configuration configuration = HBaseConfiguration.create(); 466 configuration.setBoolean("hbase.defaults.for.version.skip", true); 467 try (Connection connection = ConnectionFactory.createConnection(configuration)) { 468 /* 469 * Used to generate an overlap. 470 */ 471 Get g = new Get(Bytes.toBytes("t2,40,1564119846424.1db8c57d64e0733e0f027aaeae7a0bf0.")); 472 g.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); 473 try (Table t = connection.getTable(TableName.META_TABLE_NAME)) { 474 Result r = t.get(g); 475 byte[] row = g.getRow(); 476 row[row.length - 2] <<= row[row.length - 2]; 477 Put p = new Put(g.getRow()); 478 p.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, 479 r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER)); 480 t.put(p); 481 } 482 MetaTableAccessor.scanMetaForTableRegions(connection, visitor, null); 483 CatalogJanitorReport report = visitor.getReport(); 484 LOG.info(report != null ? report.toString() : "empty"); 485 } 486 } 487}