001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.assignment; 019 020import edu.umd.cs.findbugs.annotations.NonNull; 021import java.io.IOException; 022import java.util.ArrayList; 023import java.util.Collection; 024import java.util.Collections; 025import java.util.HashMap; 026import java.util.HashSet; 027import java.util.List; 028import java.util.Map; 029import java.util.Set; 030import java.util.concurrent.Future; 031import java.util.concurrent.TimeUnit; 032import java.util.concurrent.atomic.AtomicBoolean; 033import java.util.concurrent.locks.Condition; 034import java.util.concurrent.locks.ReentrantLock; 035import java.util.stream.Collectors; 036import java.util.stream.Stream; 037import org.apache.hadoop.conf.Configuration; 038import org.apache.hadoop.hbase.CatalogFamilyFormat; 039import org.apache.hadoop.hbase.DoNotRetryIOException; 040import org.apache.hadoop.hbase.HBaseIOException; 041import org.apache.hadoop.hbase.HConstants; 042import org.apache.hadoop.hbase.PleaseHoldException; 043import org.apache.hadoop.hbase.ServerName; 044import org.apache.hadoop.hbase.TableName; 045import org.apache.hadoop.hbase.UnknownRegionException; 046import org.apache.hadoop.hbase.client.DoNotRetryRegionException; 047import org.apache.hadoop.hbase.client.MasterSwitchType; 048import org.apache.hadoop.hbase.client.RegionInfo; 049import org.apache.hadoop.hbase.client.RegionInfoBuilder; 050import org.apache.hadoop.hbase.client.RegionReplicaUtil; 051import org.apache.hadoop.hbase.client.RegionStatesCount; 052import org.apache.hadoop.hbase.client.Result; 053import org.apache.hadoop.hbase.client.ResultScanner; 054import org.apache.hadoop.hbase.client.Scan; 055import org.apache.hadoop.hbase.client.TableDescriptor; 056import org.apache.hadoop.hbase.client.TableState; 057import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; 058import org.apache.hadoop.hbase.favored.FavoredNodesManager; 059import org.apache.hadoop.hbase.favored.FavoredNodesPromoter; 060import org.apache.hadoop.hbase.master.LoadBalancer; 061import org.apache.hadoop.hbase.master.MasterServices; 062import org.apache.hadoop.hbase.master.MetricsAssignmentManager; 063import org.apache.hadoop.hbase.master.RegionPlan; 064import org.apache.hadoop.hbase.master.RegionState; 065import org.apache.hadoop.hbase.master.RegionState.State; 066import org.apache.hadoop.hbase.master.ServerManager; 067import org.apache.hadoop.hbase.master.TableStateManager; 068import org.apache.hadoop.hbase.master.balancer.FavoredStochasticBalancer; 069import org.apache.hadoop.hbase.master.procedure.HBCKServerCrashProcedure; 070import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 071import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler; 072import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; 073import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; 074import org.apache.hadoop.hbase.master.region.MasterRegion; 075import org.apache.hadoop.hbase.procedure2.Procedure; 076import org.apache.hadoop.hbase.procedure2.ProcedureEvent; 077import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 078import org.apache.hadoop.hbase.procedure2.ProcedureInMemoryChore; 079import org.apache.hadoop.hbase.procedure2.util.StringUtils; 080import org.apache.hadoop.hbase.regionserver.SequenceId; 081import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer; 082import org.apache.hadoop.hbase.util.Bytes; 083import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 084import org.apache.hadoop.hbase.util.Pair; 085import org.apache.hadoop.hbase.util.Threads; 086import org.apache.hadoop.hbase.util.VersionInfo; 087import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 088import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 089import org.apache.yetus.audience.InterfaceAudience; 090import org.apache.zookeeper.KeeperException; 091import org.slf4j.Logger; 092import org.slf4j.LoggerFactory; 093 094import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 095import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition; 096import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; 097import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest; 098import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse; 099 100/** 101 * The AssignmentManager is the coordinator for region assign/unassign operations. 102 * <ul> 103 * <li>In-memory states of regions and servers are stored in {@link RegionStates}.</li> 104 * <li>hbase:meta state updates are handled by {@link RegionStateStore}.</li> 105 * </ul> 106 * Regions are created by CreateTable, Split, Merge. Regions are deleted by DeleteTable, Split, 107 * Merge. Assigns are triggered by CreateTable, EnableTable, Split, Merge, ServerCrash. Unassigns 108 * are triggered by DisableTable, Split, Merge 109 */ 110@InterfaceAudience.Private 111public class AssignmentManager { 112 private static final Logger LOG = LoggerFactory.getLogger(AssignmentManager.class); 113 114 // TODO: AMv2 115 // - handle region migration from hbase1 to hbase2. 116 // - handle sys table assignment first (e.g. acl, namespace) 117 // - handle table priorities 118 // - If ServerBusyException trying to update hbase:meta, we abort the Master 119 // See updateRegionLocation in RegionStateStore. 120 // 121 // See also 122 // https://docs.google.com/document/d/1eVKa7FHdeoJ1-9o8yZcOTAQbv0u0bblBlCCzVSIn69g/edit#heading=h.ystjyrkbtoq5 123 // for other TODOs. 124 125 public static final String BOOTSTRAP_THREAD_POOL_SIZE_CONF_KEY = 126 "hbase.assignment.bootstrap.thread.pool.size"; 127 128 public static final String ASSIGN_DISPATCH_WAIT_MSEC_CONF_KEY = 129 "hbase.assignment.dispatch.wait.msec"; 130 private static final int DEFAULT_ASSIGN_DISPATCH_WAIT_MSEC = 150; 131 132 public static final String ASSIGN_DISPATCH_WAITQ_MAX_CONF_KEY = 133 "hbase.assignment.dispatch.wait.queue.max.size"; 134 private static final int DEFAULT_ASSIGN_DISPATCH_WAITQ_MAX = 100; 135 136 public static final String RIT_CHORE_INTERVAL_MSEC_CONF_KEY = 137 "hbase.assignment.rit.chore.interval.msec"; 138 private static final int DEFAULT_RIT_CHORE_INTERVAL_MSEC = 60 * 1000; 139 140 public static final String DEAD_REGION_METRIC_CHORE_INTERVAL_MSEC_CONF_KEY = 141 "hbase.assignment.dead.region.metric.chore.interval.msec"; 142 private static final int DEFAULT_DEAD_REGION_METRIC_CHORE_INTERVAL_MSEC = 120 * 1000; 143 144 public static final String ASSIGN_MAX_ATTEMPTS = "hbase.assignment.maximum.attempts"; 145 private static final int DEFAULT_ASSIGN_MAX_ATTEMPTS = Integer.MAX_VALUE; 146 147 public static final String ASSIGN_RETRY_IMMEDIATELY_MAX_ATTEMPTS = 148 "hbase.assignment.retry.immediately.maximum.attempts"; 149 private static final int DEFAULT_ASSIGN_RETRY_IMMEDIATELY_MAX_ATTEMPTS = 3; 150 151 /** Region in Transition metrics threshold time */ 152 public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD = 153 "hbase.metrics.rit.stuck.warning.threshold"; 154 private static final int DEFAULT_RIT_STUCK_WARNING_THRESHOLD = 60 * 1000; 155 public static final String UNEXPECTED_STATE_REGION = "Unexpected state for "; 156 157 public static final String FORCE_REGION_RETAINMENT = "hbase.master.scp.retain.assignment.force"; 158 159 public static final boolean DEFAULT_FORCE_REGION_RETAINMENT = false; 160 161 /** The wait time in millis before checking again if the region's previous RS is back online */ 162 public static final String FORCE_REGION_RETAINMENT_WAIT_INTERVAL = 163 "hbase.master.scp.retain.assignment.force.wait-interval"; 164 165 public static final long DEFAULT_FORCE_REGION_RETAINMENT_WAIT_INTERVAL = 50; 166 167 /** 168 * The number of times to check if the region's previous RS is back online, before giving up and 169 * proceeding with assignment on a new RS 170 */ 171 public static final String FORCE_REGION_RETAINMENT_RETRIES = 172 "hbase.master.scp.retain.assignment.force.retries"; 173 174 public static final int DEFAULT_FORCE_REGION_RETAINMENT_RETRIES = 600; 175 176 private final ProcedureEvent<?> metaAssignEvent = new ProcedureEvent<>("meta assign"); 177 private final ProcedureEvent<?> metaLoadEvent = new ProcedureEvent<>("meta load"); 178 179 private final MetricsAssignmentManager metrics; 180 private final RegionInTransitionChore ritChore; 181 private final DeadServerMetricRegionChore deadMetricChore; 182 private final MasterServices master; 183 184 private final AtomicBoolean running = new AtomicBoolean(false); 185 private final RegionStates regionStates = new RegionStates(); 186 private final RegionStateStore regionStateStore; 187 188 /** 189 * When the operator uses this configuration option, any version between the current cluster 190 * version and the value of "hbase.min.version.move.system.tables" does not trigger any 191 * auto-region movement. Auto-region movement here refers to auto-migration of system table 192 * regions to newer server versions. It is assumed that the configured range of versions does not 193 * require special handling of moving system table regions to higher versioned RegionServer. This 194 * auto-migration is done by {@link #checkIfShouldMoveSystemRegionAsync()}. Example: Let's assume 195 * the cluster is on version 1.4.0 and we have set "hbase.min.version.move.system.tables" as 196 * "2.0.0". Now if we upgrade one RegionServer on 1.4.0 cluster to 1.6.0 (< 2.0.0), then 197 * AssignmentManager will not move hbase:meta, hbase:namespace and other system table regions to 198 * newly brought up RegionServer 1.6.0 as part of auto-migration. However, if we upgrade one 199 * RegionServer on 1.4.0 cluster to 2.2.0 (> 2.0.0), then AssignmentManager will move all system 200 * table regions to newly brought up RegionServer 2.2.0 as part of auto-migration done by 201 * {@link #checkIfShouldMoveSystemRegionAsync()}. "hbase.min.version.move.system.tables" is 202 * introduced as part of HBASE-22923. 203 */ 204 private final String minVersionToMoveSysTables; 205 206 private static final String MIN_VERSION_MOVE_SYS_TABLES_CONFIG = 207 "hbase.min.version.move.system.tables"; 208 private static final String DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG = ""; 209 210 private final Map<ServerName, Set<byte[]>> rsReports = new HashMap<>(); 211 212 private final boolean shouldAssignRegionsWithFavoredNodes; 213 private final int assignDispatchWaitQueueMaxSize; 214 private final int assignDispatchWaitMillis; 215 private final int assignMaxAttempts; 216 private final int assignRetryImmediatelyMaxAttempts; 217 218 private final MasterRegion masterRegion; 219 220 private final Object checkIfShouldMoveSystemRegionLock = new Object(); 221 222 private Thread assignThread; 223 224 private final boolean forceRegionRetainment; 225 226 private final long forceRegionRetainmentWaitInterval; 227 228 private final int forceRegionRetainmentRetries; 229 230 public AssignmentManager(MasterServices master, MasterRegion masterRegion) { 231 this(master, masterRegion, new RegionStateStore(master, masterRegion)); 232 } 233 234 AssignmentManager(MasterServices master, MasterRegion masterRegion, RegionStateStore stateStore) { 235 this.master = master; 236 this.regionStateStore = stateStore; 237 this.metrics = new MetricsAssignmentManager(); 238 this.masterRegion = masterRegion; 239 240 final Configuration conf = master.getConfiguration(); 241 242 // Only read favored nodes if using the favored nodes load balancer. 243 this.shouldAssignRegionsWithFavoredNodes = FavoredStochasticBalancer.class 244 .isAssignableFrom(conf.getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, Object.class)); 245 246 this.assignDispatchWaitMillis = 247 conf.getInt(ASSIGN_DISPATCH_WAIT_MSEC_CONF_KEY, DEFAULT_ASSIGN_DISPATCH_WAIT_MSEC); 248 this.assignDispatchWaitQueueMaxSize = 249 conf.getInt(ASSIGN_DISPATCH_WAITQ_MAX_CONF_KEY, DEFAULT_ASSIGN_DISPATCH_WAITQ_MAX); 250 251 this.assignMaxAttempts = 252 Math.max(1, conf.getInt(ASSIGN_MAX_ATTEMPTS, DEFAULT_ASSIGN_MAX_ATTEMPTS)); 253 this.assignRetryImmediatelyMaxAttempts = conf.getInt(ASSIGN_RETRY_IMMEDIATELY_MAX_ATTEMPTS, 254 DEFAULT_ASSIGN_RETRY_IMMEDIATELY_MAX_ATTEMPTS); 255 256 int ritChoreInterval = 257 conf.getInt(RIT_CHORE_INTERVAL_MSEC_CONF_KEY, DEFAULT_RIT_CHORE_INTERVAL_MSEC); 258 this.ritChore = new RegionInTransitionChore(ritChoreInterval); 259 260 int deadRegionChoreInterval = conf.getInt(DEAD_REGION_METRIC_CHORE_INTERVAL_MSEC_CONF_KEY, 261 DEFAULT_DEAD_REGION_METRIC_CHORE_INTERVAL_MSEC); 262 if (deadRegionChoreInterval > 0) { 263 this.deadMetricChore = new DeadServerMetricRegionChore(deadRegionChoreInterval); 264 } else { 265 this.deadMetricChore = null; 266 } 267 minVersionToMoveSysTables = 268 conf.get(MIN_VERSION_MOVE_SYS_TABLES_CONFIG, DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG); 269 270 forceRegionRetainment = 271 conf.getBoolean(FORCE_REGION_RETAINMENT, DEFAULT_FORCE_REGION_RETAINMENT); 272 forceRegionRetainmentWaitInterval = conf.getLong(FORCE_REGION_RETAINMENT_WAIT_INTERVAL, 273 DEFAULT_FORCE_REGION_RETAINMENT_WAIT_INTERVAL); 274 forceRegionRetainmentRetries = 275 conf.getInt(FORCE_REGION_RETAINMENT_RETRIES, DEFAULT_FORCE_REGION_RETAINMENT_RETRIES); 276 } 277 278 private void mirrorMetaLocations() throws IOException, KeeperException { 279 // For compatibility, mirror the meta region state to zookeeper 280 // And we still need to use zookeeper to publish the meta region locations to region 281 // server, so they can serve as ClientMetaService 282 ZKWatcher zk = master.getZooKeeper(); 283 if (zk == null || !zk.getRecoverableZooKeeper().getState().isAlive()) { 284 // this is possible in tests, we do not provide a zk watcher or the zk watcher has been closed 285 return; 286 } 287 Collection<RegionStateNode> metaStates = regionStates.getRegionStateNodes(); 288 for (RegionStateNode metaState : metaStates) { 289 MetaTableLocator.setMetaLocation(zk, metaState.getRegionLocation(), 290 metaState.getRegionInfo().getReplicaId(), metaState.getState()); 291 } 292 int replicaCount = metaStates.size(); 293 // remove extra mirror locations 294 for (String znode : zk.getMetaReplicaNodes()) { 295 int replicaId = zk.getZNodePaths().getMetaReplicaIdFromZNode(znode); 296 if (replicaId >= replicaCount) { 297 MetaTableLocator.deleteMetaLocation(zk, replicaId); 298 } 299 } 300 } 301 302 public void start() throws IOException, KeeperException { 303 if (!running.compareAndSet(false, true)) { 304 return; 305 } 306 307 LOG.trace("Starting assignment manager"); 308 309 // Start the Assignment Thread 310 startAssignmentThread(); 311 // load meta region states. 312 // here we are still in the early steps of active master startup. There is only one thread(us) 313 // can access AssignmentManager and create region node, so here we do not need to lock the 314 // region node. 315 try (ResultScanner scanner = 316 masterRegion.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) { 317 for (;;) { 318 Result result = scanner.next(); 319 if (result == null) { 320 break; 321 } 322 RegionStateStore 323 .visitMetaEntry((r, regionInfo, state, regionLocation, lastHost, openSeqNum) -> { 324 RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo); 325 regionNode.setState(state); 326 regionNode.setLastHost(lastHost); 327 regionNode.setRegionLocation(regionLocation); 328 regionNode.setOpenSeqNum(openSeqNum); 329 if (regionNode.getProcedure() != null) { 330 regionNode.getProcedure().stateLoaded(this, regionNode); 331 } 332 if (regionLocation != null) { 333 regionStates.addRegionToServer(regionNode); 334 } 335 if (RegionReplicaUtil.isDefaultReplica(regionInfo.getReplicaId())) { 336 setMetaAssigned(regionInfo, state == State.OPEN); 337 } 338 LOG.debug("Loaded hbase:meta {}", regionNode); 339 }, result); 340 } 341 } 342 mirrorMetaLocations(); 343 } 344 345 /** 346 * Create RegionStateNode based on the TRSP list, and attach the TRSP to the RegionStateNode. 347 * <p> 348 * This is used to restore the RIT region list, so we do not need to restore it in the loadingMeta 349 * method below. And it is also very important as now before submitting a TRSP, we need to attach 350 * it to the RegionStateNode, which acts like a guard, so we need to restore this information at 351 * the very beginning, before we start processing any procedures. 352 */ 353 public void setupRIT(List<TransitRegionStateProcedure> procs) { 354 procs.forEach(proc -> { 355 RegionInfo regionInfo = proc.getRegion(); 356 RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo); 357 TransitRegionStateProcedure existingProc = regionNode.getProcedure(); 358 if (existingProc != null) { 359 // This is possible, as we will detach the procedure from the RSN before we 360 // actually finish the procedure. This is because that, we will detach the TRSP from the RSN 361 // during execution, at that time, the procedure has not been marked as done in the pv2 362 // framework yet, so it is possible that we schedule a new TRSP immediately and when 363 // arriving here, we will find out that there are multiple TRSPs for the region. But we can 364 // make sure that, only the last one can take the charge, the previous ones should have all 365 // been finished already. So here we will compare the proc id, the greater one will win. 366 if (existingProc.getProcId() < proc.getProcId()) { 367 // the new one wins, unset and set it to the new one below 368 regionNode.unsetProcedure(existingProc); 369 } else { 370 // the old one wins, skip 371 return; 372 } 373 } 374 LOG.info("Attach {} to {} to restore RIT", proc, regionNode); 375 regionNode.setProcedure(proc); 376 }); 377 } 378 379 public void stop() { 380 if (!running.compareAndSet(true, false)) { 381 return; 382 } 383 384 LOG.info("Stopping assignment manager"); 385 386 // The AM is started before the procedure executor, 387 // but the actual work will be loaded/submitted only once we have the executor 388 final boolean hasProcExecutor = master.getMasterProcedureExecutor() != null; 389 390 // Remove the RIT chore 391 if (hasProcExecutor) { 392 master.getMasterProcedureExecutor().removeChore(this.ritChore); 393 if (this.deadMetricChore != null) { 394 master.getMasterProcedureExecutor().removeChore(this.deadMetricChore); 395 } 396 } 397 398 // Stop the Assignment Thread 399 stopAssignmentThread(); 400 401 // Stop the RegionStateStore 402 regionStates.clear(); 403 404 // Update meta events (for testing) 405 if (hasProcExecutor) { 406 metaLoadEvent.suspend(); 407 for (RegionInfo hri : getMetaRegionSet()) { 408 setMetaAssigned(hri, false); 409 } 410 } 411 } 412 413 public boolean isRunning() { 414 return running.get(); 415 } 416 417 public Configuration getConfiguration() { 418 return master.getConfiguration(); 419 } 420 421 public MetricsAssignmentManager getAssignmentManagerMetrics() { 422 return metrics; 423 } 424 425 private LoadBalancer getBalancer() { 426 return master.getLoadBalancer(); 427 } 428 429 private FavoredNodesPromoter getFavoredNodePromoter() { 430 return (FavoredNodesPromoter) ((RSGroupBasedLoadBalancer) master.getLoadBalancer()) 431 .getInternalBalancer(); 432 } 433 434 private MasterProcedureEnv getProcedureEnvironment() { 435 return master.getMasterProcedureExecutor().getEnvironment(); 436 } 437 438 private MasterProcedureScheduler getProcedureScheduler() { 439 return getProcedureEnvironment().getProcedureScheduler(); 440 } 441 442 int getAssignMaxAttempts() { 443 return assignMaxAttempts; 444 } 445 446 public boolean isForceRegionRetainment() { 447 return forceRegionRetainment; 448 } 449 450 public long getForceRegionRetainmentWaitInterval() { 451 return forceRegionRetainmentWaitInterval; 452 } 453 454 public int getForceRegionRetainmentRetries() { 455 return forceRegionRetainmentRetries; 456 } 457 458 int getAssignRetryImmediatelyMaxAttempts() { 459 return assignRetryImmediatelyMaxAttempts; 460 } 461 462 public RegionStates getRegionStates() { 463 return regionStates; 464 } 465 466 /** 467 * Returns the regions hosted by the specified server. 468 * <p/> 469 * Notice that, for SCP, after we submit the SCP, no one can change the region list for the 470 * ServerStateNode so we do not need any locks here. And for other usage, this can only give you a 471 * snapshot of the current region list for this server, which means, right after you get the 472 * region list, new regions may be moved to this server or some regions may be moved out from this 473 * server, so you should not use it critically if you need strong consistency. 474 */ 475 public List<RegionInfo> getRegionsOnServer(ServerName serverName) { 476 ServerStateNode serverInfo = regionStates.getServerNode(serverName); 477 if (serverInfo == null) { 478 return Collections.emptyList(); 479 } 480 return serverInfo.getRegionInfoList(); 481 } 482 483 private RegionInfo getRegionInfo(RegionStateNode rsn) { 484 if (rsn.isSplit() && !rsn.getRegionInfo().isSplit()) { 485 // see the comments in markRegionAsSplit on why we need to do this converting. 486 return RegionInfoBuilder.newBuilder(rsn.getRegionInfo()).setSplit(true).setOffline(true) 487 .build(); 488 } else { 489 return rsn.getRegionInfo(); 490 } 491 } 492 493 private Stream<RegionStateNode> getRegionStateNodes(TableName tableName, 494 boolean excludeOfflinedSplitParents) { 495 Stream<RegionStateNode> stream = regionStates.getTableRegionStateNodes(tableName).stream(); 496 if (excludeOfflinedSplitParents) { 497 return stream.filter(rsn -> !rsn.isSplit()); 498 } else { 499 return stream; 500 } 501 } 502 503 public List<RegionInfo> getTableRegions(TableName tableName, 504 boolean excludeOfflinedSplitParents) { 505 return getRegionStateNodes(tableName, excludeOfflinedSplitParents).map(this::getRegionInfo) 506 .collect(Collectors.toList()); 507 } 508 509 public List<Pair<RegionInfo, ServerName>> getTableRegionsAndLocations(TableName tableName, 510 boolean excludeOfflinedSplitParents) { 511 return getRegionStateNodes(tableName, excludeOfflinedSplitParents) 512 .map(rsn -> Pair.newPair(getRegionInfo(rsn), rsn.getRegionLocation())) 513 .collect(Collectors.toList()); 514 } 515 516 public RegionStateStore getRegionStateStore() { 517 return regionStateStore; 518 } 519 520 public List<ServerName> getFavoredNodes(final RegionInfo regionInfo) { 521 return this.shouldAssignRegionsWithFavoredNodes 522 ? getFavoredNodePromoter().getFavoredNodes(regionInfo) 523 : ServerName.EMPTY_SERVER_LIST; 524 } 525 526 // ============================================================================================ 527 // Table State Manager helpers 528 // ============================================================================================ 529 private TableStateManager getTableStateManager() { 530 return master.getTableStateManager(); 531 } 532 533 private boolean isTableEnabled(final TableName tableName) { 534 return getTableStateManager().isTableState(tableName, TableState.State.ENABLED); 535 } 536 537 private boolean isTableDisabled(final TableName tableName) { 538 return getTableStateManager().isTableState(tableName, TableState.State.DISABLED, 539 TableState.State.DISABLING); 540 } 541 542 // ============================================================================================ 543 // META Helpers 544 // ============================================================================================ 545 private boolean isMetaRegion(final RegionInfo regionInfo) { 546 return regionInfo.isMetaRegion(); 547 } 548 549 public boolean isMetaRegion(final byte[] regionName) { 550 return getMetaRegionFromName(regionName) != null; 551 } 552 553 public RegionInfo getMetaRegionFromName(final byte[] regionName) { 554 for (RegionInfo hri : getMetaRegionSet()) { 555 if (Bytes.equals(hri.getRegionName(), regionName)) { 556 return hri; 557 } 558 } 559 return null; 560 } 561 562 public boolean isCarryingMeta(final ServerName serverName) { 563 // TODO: handle multiple meta 564 return isCarryingRegion(serverName, RegionInfoBuilder.FIRST_META_REGIONINFO); 565 } 566 567 private boolean isCarryingRegion(final ServerName serverName, final RegionInfo regionInfo) { 568 // TODO: check for state? 569 final RegionStateNode node = regionStates.getRegionStateNode(regionInfo); 570 return (node != null && serverName.equals(node.getRegionLocation())); 571 } 572 573 private RegionInfo getMetaForRegion(final RegionInfo regionInfo) { 574 // if (regionInfo.isMetaRegion()) return regionInfo; 575 // TODO: handle multiple meta. if the region provided is not meta lookup 576 // which meta the region belongs to. 577 return RegionInfoBuilder.FIRST_META_REGIONINFO; 578 } 579 580 // TODO: handle multiple meta. 581 private static final Set<RegionInfo> META_REGION_SET = 582 Collections.singleton(RegionInfoBuilder.FIRST_META_REGIONINFO); 583 584 public Set<RegionInfo> getMetaRegionSet() { 585 return META_REGION_SET; 586 } 587 588 // ============================================================================================ 589 // META Event(s) helpers 590 // ============================================================================================ 591 /** 592 * Notice that, this only means the meta region is available on a RS, but the AM may still be 593 * loading the region states from meta, so usually you need to check {@link #isMetaLoaded()} first 594 * before checking this method, unless you can make sure that your piece of code can only be 595 * executed after AM builds the region states. 596 * @see #isMetaLoaded() 597 */ 598 public boolean isMetaAssigned() { 599 return metaAssignEvent.isReady(); 600 } 601 602 public boolean isMetaRegionInTransition() { 603 return !isMetaAssigned(); 604 } 605 606 /** 607 * Notice that this event does not mean the AM has already finished region state rebuilding. See 608 * the comment of {@link #isMetaAssigned()} for more details. 609 * @see #isMetaAssigned() 610 */ 611 public boolean waitMetaAssigned(Procedure<?> proc, RegionInfo regionInfo) { 612 return getMetaAssignEvent(getMetaForRegion(regionInfo)).suspendIfNotReady(proc); 613 } 614 615 private void setMetaAssigned(RegionInfo metaRegionInfo, boolean assigned) { 616 assert isMetaRegion(metaRegionInfo) : "unexpected non-meta region " + metaRegionInfo; 617 ProcedureEvent<?> metaAssignEvent = getMetaAssignEvent(metaRegionInfo); 618 if (assigned) { 619 metaAssignEvent.wake(getProcedureScheduler()); 620 } else { 621 metaAssignEvent.suspend(); 622 } 623 } 624 625 private ProcedureEvent<?> getMetaAssignEvent(RegionInfo metaRegionInfo) { 626 assert isMetaRegion(metaRegionInfo) : "unexpected non-meta region " + metaRegionInfo; 627 // TODO: handle multiple meta. 628 return metaAssignEvent; 629 } 630 631 /** 632 * Wait until AM finishes the meta loading, i.e, the region states rebuilding. 633 * @see #isMetaLoaded() 634 * @see #waitMetaAssigned(Procedure, RegionInfo) 635 */ 636 public boolean waitMetaLoaded(Procedure<?> proc) { 637 return metaLoadEvent.suspendIfNotReady(proc); 638 } 639 640 /** 641 * This method will be called in master initialization method after calling 642 * {@link #processOfflineRegions()}, as in processOfflineRegions we will generate assign 643 * procedures for offline regions, which may be conflict with creating table. 644 * <p/> 645 * This is a bit dirty, should be reconsidered after we decide whether to keep the 646 * {@link #processOfflineRegions()} method. 647 */ 648 public void wakeMetaLoadedEvent() { 649 metaLoadEvent.wake(getProcedureScheduler()); 650 assert isMetaLoaded() : "expected meta to be loaded"; 651 } 652 653 /** 654 * Return whether AM finishes the meta loading, i.e, the region states rebuilding. 655 * @see #isMetaAssigned() 656 * @see #waitMetaLoaded(Procedure) 657 */ 658 public boolean isMetaLoaded() { 659 return metaLoadEvent.isReady(); 660 } 661 662 /** 663 * Start a new thread to check if there are region servers whose versions are higher than others. 664 * If so, move all system table regions to RS with the highest version to keep compatibility. The 665 * reason is, RS in new version may not be able to access RS in old version when there are some 666 * incompatible changes. 667 * <p> 668 * This method is called when a new RegionServer is added to cluster only. 669 * </p> 670 */ 671 public void checkIfShouldMoveSystemRegionAsync() { 672 // TODO: Fix this thread. If a server is killed and a new one started, this thread thinks that 673 // it should 'move' the system tables from the old server to the new server but 674 // ServerCrashProcedure is on it; and it will take care of the assign without dataloss. 675 if (this.master.getServerManager().countOfRegionServers() <= 1) { 676 return; 677 } 678 // This thread used to run whenever there was a change in the cluster. The ZooKeeper 679 // childrenChanged notification came in before the nodeDeleted message and so this method 680 // cold run before a ServerCrashProcedure could run. That meant that this thread could see 681 // a Crashed Server before ServerCrashProcedure and it could find system regions on the 682 // crashed server and go move them before ServerCrashProcedure had a chance; could be 683 // dataloss too if WALs were not recovered. 684 new Thread(() -> { 685 try { 686 synchronized (checkIfShouldMoveSystemRegionLock) { 687 List<RegionPlan> plans = new ArrayList<>(); 688 // TODO: I don't think this code does a good job if all servers in cluster have same 689 // version. It looks like it will schedule unnecessary moves. 690 for (ServerName server : getExcludedServersForSystemTable()) { 691 if (master.getServerManager().isServerDead(server)) { 692 // TODO: See HBASE-18494 and HBASE-18495. Though getExcludedServersForSystemTable() 693 // considers only online servers, the server could be queued for dead server 694 // processing. As region assignments for crashed server is handled by 695 // ServerCrashProcedure, do NOT handle them here. The goal is to handle this through 696 // regular flow of LoadBalancer as a favored node and not to have this special 697 // handling. 698 continue; 699 } 700 List<RegionInfo> regionsShouldMove = getSystemTables(server); 701 if (!regionsShouldMove.isEmpty()) { 702 for (RegionInfo regionInfo : regionsShouldMove) { 703 // null value for dest forces destination server to be selected by balancer 704 RegionPlan plan = new RegionPlan(regionInfo, server, null); 705 if (regionInfo.isMetaRegion()) { 706 // Must move meta region first. 707 LOG.info("Async MOVE of {} to newer Server={}", regionInfo.getEncodedName(), 708 server); 709 moveAsync(plan); 710 } else { 711 plans.add(plan); 712 } 713 } 714 } 715 for (RegionPlan plan : plans) { 716 LOG.info("Async MOVE of {} to newer Server={}", plan.getRegionInfo().getEncodedName(), 717 server); 718 moveAsync(plan); 719 } 720 } 721 } 722 } catch (Throwable t) { 723 LOG.error(t.toString(), t); 724 } 725 }).start(); 726 } 727 728 private List<RegionInfo> getSystemTables(ServerName serverName) { 729 ServerStateNode serverNode = regionStates.getServerNode(serverName); 730 if (serverNode == null) { 731 return Collections.emptyList(); 732 } 733 return serverNode.getSystemRegionInfoList(); 734 } 735 736 private void preTransitCheck(RegionStateNode regionNode, RegionState.State[] expectedStates) 737 throws HBaseIOException { 738 if (regionNode.getProcedure() != null) { 739 throw new HBaseIOException( 740 regionNode + " is currently in transition; pid=" + regionNode.getProcedure().getProcId()); 741 } 742 if (!regionNode.isInState(expectedStates)) { 743 throw new DoNotRetryRegionException(UNEXPECTED_STATE_REGION + regionNode); 744 } 745 if (isTableDisabled(regionNode.getTable())) { 746 throw new DoNotRetryIOException(regionNode.getTable() + " is disabled for " + regionNode); 747 } 748 } 749 750 /** 751 * Create an assign TransitRegionStateProcedure. Makes sure of RegionState. Throws exception if 752 * not appropriate UNLESS override is set. Used by hbck2 but also by straightline 753 * {@link #assign(RegionInfo, ServerName)} and {@link #assignAsync(RegionInfo, ServerName)}. 754 * @see #createAssignProcedure(RegionStateNode, ServerName) for a version that does NO checking 755 * used when only when no checking needed. 756 * @param override If false, check RegionState is appropriate for assign; if not throw exception. 757 */ 758 private TransitRegionStateProcedure createAssignProcedure(RegionInfo regionInfo, ServerName sn, 759 boolean override) throws IOException { 760 RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo); 761 regionNode.lock(); 762 try { 763 if (override) { 764 if (regionNode.getProcedure() != null) { 765 regionNode.unsetProcedure(regionNode.getProcedure()); 766 } 767 } else { 768 preTransitCheck(regionNode, STATES_EXPECTED_ON_ASSIGN); 769 } 770 assert regionNode.getProcedure() == null; 771 return regionNode.setProcedure( 772 TransitRegionStateProcedure.assign(getProcedureEnvironment(), regionInfo, sn)); 773 } finally { 774 regionNode.unlock(); 775 } 776 } 777 778 /** 779 * Create an assign TransitRegionStateProcedure. Does NO checking of RegionState. Presumes 780 * appriopriate state ripe for assign. 781 * @see #createAssignProcedure(RegionInfo, ServerName, boolean) 782 */ 783 private TransitRegionStateProcedure createAssignProcedure(RegionStateNode regionNode, 784 ServerName targetServer) { 785 regionNode.lock(); 786 try { 787 return regionNode.setProcedure(TransitRegionStateProcedure.assign(getProcedureEnvironment(), 788 regionNode.getRegionInfo(), targetServer)); 789 } finally { 790 regionNode.unlock(); 791 } 792 } 793 794 public long assign(RegionInfo regionInfo, ServerName sn) throws IOException { 795 TransitRegionStateProcedure proc = createAssignProcedure(regionInfo, sn, false); 796 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); 797 return proc.getProcId(); 798 } 799 800 public long assign(RegionInfo regionInfo) throws IOException { 801 return assign(regionInfo, null); 802 } 803 804 /** 805 * Submits a procedure that assigns a region to a target server without waiting for it to finish 806 * @param regionInfo the region we would like to assign 807 * @param sn target server name 808 */ 809 public Future<byte[]> assignAsync(RegionInfo regionInfo, ServerName sn) throws IOException { 810 return ProcedureSyncWait.submitProcedure(master.getMasterProcedureExecutor(), 811 createAssignProcedure(regionInfo, sn, false)); 812 } 813 814 /** 815 * Submits a procedure that assigns a region without waiting for it to finish 816 * @param regionInfo the region we would like to assign 817 */ 818 public Future<byte[]> assignAsync(RegionInfo regionInfo) throws IOException { 819 return assignAsync(regionInfo, null); 820 } 821 822 public long unassign(RegionInfo regionInfo) throws IOException { 823 RegionStateNode regionNode = regionStates.getRegionStateNode(regionInfo); 824 if (regionNode == null) { 825 throw new UnknownRegionException("No RegionState found for " + regionInfo.getEncodedName()); 826 } 827 TransitRegionStateProcedure proc; 828 regionNode.lock(); 829 try { 830 preTransitCheck(regionNode, STATES_EXPECTED_ON_UNASSIGN_OR_MOVE); 831 proc = TransitRegionStateProcedure.unassign(getProcedureEnvironment(), regionInfo); 832 regionNode.setProcedure(proc); 833 } finally { 834 regionNode.unlock(); 835 } 836 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); 837 return proc.getProcId(); 838 } 839 840 public TransitRegionStateProcedure createMoveRegionProcedure(RegionInfo regionInfo, 841 ServerName targetServer) throws HBaseIOException { 842 RegionStateNode regionNode = this.regionStates.getRegionStateNode(regionInfo); 843 if (regionNode == null) { 844 throw new UnknownRegionException( 845 "No RegionStateNode found for " + regionInfo.getEncodedName() + "(Closed/Deleted?)"); 846 } 847 TransitRegionStateProcedure proc; 848 regionNode.lock(); 849 try { 850 preTransitCheck(regionNode, STATES_EXPECTED_ON_UNASSIGN_OR_MOVE); 851 regionNode.checkOnline(); 852 proc = TransitRegionStateProcedure.move(getProcedureEnvironment(), regionInfo, targetServer); 853 regionNode.setProcedure(proc); 854 } finally { 855 regionNode.unlock(); 856 } 857 return proc; 858 } 859 860 public void move(RegionInfo regionInfo) throws IOException { 861 TransitRegionStateProcedure proc = createMoveRegionProcedure(regionInfo, null); 862 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); 863 } 864 865 public Future<byte[]> moveAsync(RegionPlan regionPlan) throws HBaseIOException { 866 TransitRegionStateProcedure proc = 867 createMoveRegionProcedure(regionPlan.getRegionInfo(), regionPlan.getDestination()); 868 return ProcedureSyncWait.submitProcedure(master.getMasterProcedureExecutor(), proc); 869 } 870 871 public Future<byte[]> balance(RegionPlan regionPlan) throws HBaseIOException { 872 ServerName current = 873 this.getRegionStates().getRegionAssignments().get(regionPlan.getRegionInfo()); 874 if (current == null || !current.equals(regionPlan.getSource())) { 875 LOG.debug("Skip region plan {}, source server not match, current region location is {}", 876 regionPlan, current == null ? "(null)" : current); 877 return null; 878 } 879 return moveAsync(regionPlan); 880 } 881 882 // ============================================================================================ 883 // RegionTransition procedures helpers 884 // ============================================================================================ 885 886 /** 887 * Create round-robin assigns. Use on table creation to distribute out regions across cluster. 888 * @return AssignProcedures made out of the passed in <code>hris</code> and a call to the balancer 889 * to populate the assigns with targets chosen using round-robin (default balancer 890 * scheme). If at assign-time, the target chosen is no longer up, thats fine, the 891 * AssignProcedure will ask the balancer for a new target, and so on. 892 */ 893 public TransitRegionStateProcedure[] createRoundRobinAssignProcedures(List<RegionInfo> hris, 894 List<ServerName> serversToExclude) { 895 if (hris.isEmpty()) { 896 return new TransitRegionStateProcedure[0]; 897 } 898 899 if ( 900 serversToExclude != null && this.master.getServerManager().getOnlineServersList().size() == 1 901 ) { 902 LOG.debug("Only one region server found and hence going ahead with the assignment"); 903 serversToExclude = null; 904 } 905 try { 906 // Ask the balancer to assign our regions. Pass the regions en masse. The balancer can do 907 // a better job if it has all the assignments in the one lump. 908 Map<ServerName, List<RegionInfo>> assignments = getBalancer().roundRobinAssignment(hris, 909 this.master.getServerManager().createDestinationServersList(serversToExclude)); 910 // Return mid-method! 911 return createAssignProcedures(assignments); 912 } catch (IOException hioe) { 913 LOG.warn("Failed roundRobinAssignment", hioe); 914 } 915 // If an error above, fall-through to this simpler assign. Last resort. 916 return createAssignProcedures(hris); 917 } 918 919 /** 920 * Create round-robin assigns. Use on table creation to distribute out regions across cluster. 921 * @return AssignProcedures made out of the passed in <code>hris</code> and a call to the balancer 922 * to populate the assigns with targets chosen using round-robin (default balancer 923 * scheme). If at assign-time, the target chosen is no longer up, thats fine, the 924 * AssignProcedure will ask the balancer for a new target, and so on. 925 */ 926 public TransitRegionStateProcedure[] createRoundRobinAssignProcedures(List<RegionInfo> hris) { 927 return createRoundRobinAssignProcedures(hris, null); 928 } 929 930 static int compare(TransitRegionStateProcedure left, TransitRegionStateProcedure right) { 931 if (left.getRegion().isMetaRegion()) { 932 if (right.getRegion().isMetaRegion()) { 933 return RegionInfo.COMPARATOR.compare(left.getRegion(), right.getRegion()); 934 } 935 return -1; 936 } else if (right.getRegion().isMetaRegion()) { 937 return +1; 938 } 939 if (left.getRegion().getTable().isSystemTable()) { 940 if (right.getRegion().getTable().isSystemTable()) { 941 return RegionInfo.COMPARATOR.compare(left.getRegion(), right.getRegion()); 942 } 943 return -1; 944 } else if (right.getRegion().getTable().isSystemTable()) { 945 return +1; 946 } 947 return RegionInfo.COMPARATOR.compare(left.getRegion(), right.getRegion()); 948 } 949 950 /** 951 * Create one TransitRegionStateProcedure to assign a region w/o specifying a target server. This 952 * method is called from HBCK2. 953 * @return an assign or null 954 */ 955 public TransitRegionStateProcedure createOneAssignProcedure(RegionInfo ri, boolean override) { 956 TransitRegionStateProcedure trsp = null; 957 try { 958 trsp = createAssignProcedure(ri, null, override); 959 } catch (IOException ioe) { 960 LOG.info( 961 "Failed {} assign, override={}" 962 + (override ? "" : "; set override to by-pass state checks."), 963 ri.getEncodedName(), override, ioe); 964 } 965 return trsp; 966 } 967 968 /** 969 * Create one TransitRegionStateProcedure to unassign a region. This method is called from HBCK2. 970 * @return an unassign or null 971 */ 972 public TransitRegionStateProcedure createOneUnassignProcedure(RegionInfo ri, boolean override) { 973 RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(ri); 974 TransitRegionStateProcedure trsp = null; 975 regionNode.lock(); 976 try { 977 if (override) { 978 if (regionNode.getProcedure() != null) { 979 regionNode.unsetProcedure(regionNode.getProcedure()); 980 } 981 } else { 982 // This is where we could throw an exception; i.e. override is false. 983 preTransitCheck(regionNode, STATES_EXPECTED_ON_UNASSIGN_OR_MOVE); 984 } 985 assert regionNode.getProcedure() == null; 986 trsp = 987 TransitRegionStateProcedure.unassign(getProcedureEnvironment(), regionNode.getRegionInfo()); 988 regionNode.setProcedure(trsp); 989 } catch (IOException ioe) { 990 // 'override' must be false here. 991 LOG.info("Failed {} unassign, override=false; set override to by-pass state checks.", 992 ri.getEncodedName(), ioe); 993 } finally { 994 regionNode.unlock(); 995 } 996 return trsp; 997 } 998 999 /** 1000 * Create an array of TransitRegionStateProcedure w/o specifying a target server. Used as fallback 1001 * of caller is unable to do {@link #createAssignProcedures(Map)}. 1002 * <p/> 1003 * If no target server, at assign time, we will try to use the former location of the region if 1004 * one exists. This is how we 'retain' the old location across a server restart. 1005 * <p/> 1006 * Should only be called when you can make sure that no one can touch these regions other than 1007 * you. For example, when you are creating or enabling table. Presumes all Regions are in 1008 * appropriate state ripe for assign; no checking of Region state is done in here. 1009 * @see #createAssignProcedures(Map) 1010 */ 1011 public TransitRegionStateProcedure[] createAssignProcedures(List<RegionInfo> hris) { 1012 return hris.stream().map(hri -> regionStates.getOrCreateRegionStateNode(hri)) 1013 .map(regionNode -> createAssignProcedure(regionNode, null)).sorted(AssignmentManager::compare) 1014 .toArray(TransitRegionStateProcedure[]::new); 1015 } 1016 1017 /** 1018 * Tied to {@link #createAssignProcedures(List)} in that it is called if caller is unable to run 1019 * this method. Presumes all Regions are in appropriate state ripe for assign; no checking of 1020 * Region state is done in here. 1021 * @param assignments Map of assignments from which we produce an array of AssignProcedures. 1022 * @return Assignments made from the passed in <code>assignments</code> 1023 * @see #createAssignProcedures(List) 1024 */ 1025 private TransitRegionStateProcedure[] 1026 createAssignProcedures(Map<ServerName, List<RegionInfo>> assignments) { 1027 return assignments.entrySet().stream() 1028 .flatMap(e -> e.getValue().stream().map(hri -> regionStates.getOrCreateRegionStateNode(hri)) 1029 .map(regionNode -> createAssignProcedure(regionNode, e.getKey()))) 1030 .sorted(AssignmentManager::compare).toArray(TransitRegionStateProcedure[]::new); 1031 } 1032 1033 // for creating unassign TRSP when disabling a table or closing excess region replicas 1034 private TransitRegionStateProcedure forceCreateUnssignProcedure(RegionStateNode regionNode) { 1035 regionNode.lock(); 1036 try { 1037 if (regionNode.isInState(State.OFFLINE, State.CLOSED, State.SPLIT)) { 1038 return null; 1039 } 1040 // in general, a split parent should be in CLOSED or SPLIT state, but anyway, let's check it 1041 // here for safety 1042 if (regionNode.getRegionInfo().isSplit()) { 1043 LOG.warn("{} is a split parent but not in CLOSED or SPLIT state", regionNode); 1044 return null; 1045 } 1046 // As in DisableTableProcedure or ModifyTableProcedure, we will hold the xlock for table, so 1047 // we can make sure that this procedure has not been executed yet, as TRSP will hold the 1048 // shared lock for table all the time. So here we will unset it and when it is actually 1049 // executed, it will find that the attach procedure is not itself and quit immediately. 1050 if (regionNode.getProcedure() != null) { 1051 regionNode.unsetProcedure(regionNode.getProcedure()); 1052 } 1053 return regionNode.setProcedure(TransitRegionStateProcedure.unassign(getProcedureEnvironment(), 1054 regionNode.getRegionInfo())); 1055 } finally { 1056 regionNode.unlock(); 1057 } 1058 } 1059 1060 /** 1061 * Called by DisableTableProcedure to unassign all the regions for a table. 1062 */ 1063 public TransitRegionStateProcedure[] createUnassignProceduresForDisabling(TableName tableName) { 1064 return regionStates.getTableRegionStateNodes(tableName).stream() 1065 .map(this::forceCreateUnssignProcedure).filter(p -> p != null) 1066 .toArray(TransitRegionStateProcedure[]::new); 1067 } 1068 1069 /** 1070 * Called by ModifyTableProcedures to unassign all the excess region replicas for a table. 1071 */ 1072 public TransitRegionStateProcedure[] createUnassignProceduresForClosingExcessRegionReplicas( 1073 TableName tableName, int newReplicaCount) { 1074 return regionStates.getTableRegionStateNodes(tableName).stream() 1075 .filter(regionNode -> regionNode.getRegionInfo().getReplicaId() >= newReplicaCount) 1076 .map(this::forceCreateUnssignProcedure).filter(p -> p != null) 1077 .toArray(TransitRegionStateProcedure[]::new); 1078 } 1079 1080 public SplitTableRegionProcedure createSplitProcedure(final RegionInfo regionToSplit, 1081 final byte[] splitKey) throws IOException { 1082 return new SplitTableRegionProcedure(getProcedureEnvironment(), regionToSplit, splitKey); 1083 } 1084 1085 public MergeTableRegionsProcedure createMergeProcedure(RegionInfo... ris) throws IOException { 1086 return new MergeTableRegionsProcedure(getProcedureEnvironment(), ris, false); 1087 } 1088 1089 /** 1090 * Delete the region states. This is called by "DeleteTable" 1091 */ 1092 public void deleteTable(final TableName tableName) throws IOException { 1093 final ArrayList<RegionInfo> regions = regionStates.getTableRegionsInfo(tableName); 1094 regionStateStore.deleteRegions(regions); 1095 for (int i = 0; i < regions.size(); ++i) { 1096 final RegionInfo regionInfo = regions.get(i); 1097 regionStates.deleteRegion(regionInfo); 1098 } 1099 } 1100 1101 // ============================================================================================ 1102 // RS Region Transition Report helpers 1103 // ============================================================================================ 1104 private void reportRegionStateTransition(ReportRegionStateTransitionResponse.Builder builder, 1105 ServerName serverName, List<RegionStateTransition> transitionList) throws IOException { 1106 for (RegionStateTransition transition : transitionList) { 1107 switch (transition.getTransitionCode()) { 1108 case OPENED: 1109 case FAILED_OPEN: 1110 case CLOSED: 1111 assert transition.getRegionInfoCount() == 1 : transition; 1112 final RegionInfo hri = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); 1113 long procId = 1114 transition.getProcIdCount() > 0 ? transition.getProcId(0) : Procedure.NO_PROC_ID; 1115 updateRegionTransition(serverName, transition.getTransitionCode(), hri, 1116 transition.hasOpenSeqNum() ? transition.getOpenSeqNum() : HConstants.NO_SEQNUM, procId); 1117 break; 1118 case READY_TO_SPLIT: 1119 case SPLIT: 1120 case SPLIT_REVERTED: 1121 assert transition.getRegionInfoCount() == 3 : transition; 1122 final RegionInfo parent = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); 1123 final RegionInfo splitA = ProtobufUtil.toRegionInfo(transition.getRegionInfo(1)); 1124 final RegionInfo splitB = ProtobufUtil.toRegionInfo(transition.getRegionInfo(2)); 1125 updateRegionSplitTransition(serverName, transition.getTransitionCode(), parent, splitA, 1126 splitB); 1127 break; 1128 case READY_TO_MERGE: 1129 case MERGED: 1130 case MERGE_REVERTED: 1131 assert transition.getRegionInfoCount() == 3 : transition; 1132 final RegionInfo merged = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); 1133 final RegionInfo mergeA = ProtobufUtil.toRegionInfo(transition.getRegionInfo(1)); 1134 final RegionInfo mergeB = ProtobufUtil.toRegionInfo(transition.getRegionInfo(2)); 1135 updateRegionMergeTransition(serverName, transition.getTransitionCode(), merged, mergeA, 1136 mergeB); 1137 break; 1138 } 1139 } 1140 } 1141 1142 public ReportRegionStateTransitionResponse reportRegionStateTransition( 1143 final ReportRegionStateTransitionRequest req) throws PleaseHoldException { 1144 ReportRegionStateTransitionResponse.Builder builder = 1145 ReportRegionStateTransitionResponse.newBuilder(); 1146 ServerName serverName = ProtobufUtil.toServerName(req.getServer()); 1147 ServerStateNode serverNode = regionStates.getOrCreateServer(serverName); 1148 // here we have to acquire a read lock instead of a simple exclusive lock. This is because that 1149 // we should not block other reportRegionStateTransition call from the same region server. This 1150 // is not only about performance, but also to prevent dead lock. Think of the meta region is 1151 // also on the same region server and you hold the lock which blocks the 1152 // reportRegionStateTransition for meta, and since meta is not online, you will block inside the 1153 // lock protection to wait for meta online... 1154 serverNode.readLock().lock(); 1155 try { 1156 // we only accept reportRegionStateTransition if the region server is online, see the comment 1157 // above in submitServerCrash method and HBASE-21508 for more details. 1158 if (serverNode.isInState(ServerState.ONLINE)) { 1159 try { 1160 reportRegionStateTransition(builder, serverName, req.getTransitionList()); 1161 } catch (PleaseHoldException e) { 1162 LOG.trace("Failed transition ", e); 1163 throw e; 1164 } catch (UnsupportedOperationException | IOException e) { 1165 // TODO: at the moment we have a single error message and the RS will abort 1166 // if the master says that one of the region transitions failed. 1167 LOG.warn("Failed transition", e); 1168 builder.setErrorMessage("Failed transition " + e.getMessage()); 1169 } 1170 } else { 1171 LOG.warn("The region server {} is already dead, skip reportRegionStateTransition call", 1172 serverName); 1173 builder.setErrorMessage("You are dead"); 1174 } 1175 } finally { 1176 serverNode.readLock().unlock(); 1177 } 1178 1179 return builder.build(); 1180 } 1181 1182 private void updateRegionTransition(ServerName serverName, TransitionCode state, 1183 RegionInfo regionInfo, long seqId, long procId) throws IOException { 1184 checkMetaLoaded(regionInfo); 1185 1186 RegionStateNode regionNode = regionStates.getRegionStateNode(regionInfo); 1187 if (regionNode == null) { 1188 // the table/region is gone. maybe a delete, split, merge 1189 throw new UnexpectedStateException(String.format( 1190 "Server %s was trying to transition region %s to %s. but Region is not known.", serverName, 1191 regionInfo, state)); 1192 } 1193 LOG.trace("Update region transition serverName={} region={} regionState={}", serverName, 1194 regionNode, state); 1195 1196 ServerStateNode serverNode = regionStates.getOrCreateServer(serverName); 1197 regionNode.lock(); 1198 try { 1199 if (!reportTransition(regionNode, serverNode, state, seqId, procId)) { 1200 // Don't log WARN if shutting down cluster; during shutdown. Avoid the below messages: 1201 // 2018-08-13 10:45:10,551 WARN ...AssignmentManager: No matching procedure found for 1202 // rit=OPEN, location=ve0538.halxg.cloudera.com,16020,1533493000958, 1203 // table=IntegrationTestBigLinkedList, region=65ab289e2fc1530df65f6c3d7cde7aa5 transition 1204 // to CLOSED 1205 // These happen because on cluster shutdown, we currently let the RegionServers close 1206 // regions. This is the only time that region close is not run by the Master (so cluster 1207 // goes down fast). Consider changing it so Master runs all shutdowns. 1208 if ( 1209 this.master.getServerManager().isClusterShutdown() && state.equals(TransitionCode.CLOSED) 1210 ) { 1211 LOG.info("RegionServer {} {}", state, regionNode.getRegionInfo().getEncodedName()); 1212 } else { 1213 LOG.warn("No matching procedure found for {} transition on {} to {}", serverName, 1214 regionNode, state); 1215 } 1216 } 1217 } finally { 1218 regionNode.unlock(); 1219 } 1220 } 1221 1222 private boolean reportTransition(RegionStateNode regionNode, ServerStateNode serverNode, 1223 TransitionCode state, long seqId, long procId) throws IOException { 1224 ServerName serverName = serverNode.getServerName(); 1225 TransitRegionStateProcedure proc = regionNode.getProcedure(); 1226 if (proc == null) { 1227 return false; 1228 } 1229 proc.reportTransition(master.getMasterProcedureExecutor().getEnvironment(), regionNode, 1230 serverName, state, seqId, procId); 1231 return true; 1232 } 1233 1234 private void updateRegionSplitTransition(final ServerName serverName, final TransitionCode state, 1235 final RegionInfo parent, final RegionInfo hriA, final RegionInfo hriB) throws IOException { 1236 checkMetaLoaded(parent); 1237 1238 if (state != TransitionCode.READY_TO_SPLIT) { 1239 throw new UnexpectedStateException( 1240 "unsupported split regionState=" + state + " for parent region " + parent 1241 + " maybe an old RS (< 2.0) had the operation in progress"); 1242 } 1243 1244 // sanity check on the request 1245 if (!Bytes.equals(hriA.getEndKey(), hriB.getStartKey())) { 1246 throw new UnsupportedOperationException("unsupported split request with bad keys: parent=" 1247 + parent + " hriA=" + hriA + " hriB=" + hriB); 1248 } 1249 1250 if (!master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)) { 1251 LOG.warn("Split switch is off! skip split of " + parent); 1252 throw new DoNotRetryIOException( 1253 "Split region " + parent.getRegionNameAsString() + " failed due to split switch off"); 1254 } 1255 1256 // Submit the Split procedure 1257 final byte[] splitKey = hriB.getStartKey(); 1258 if (LOG.isDebugEnabled()) { 1259 LOG.debug("Split request from " + serverName + ", parent=" + parent + " splitKey=" 1260 + Bytes.toStringBinary(splitKey)); 1261 } 1262 // Processing this report happens asynchronously from other activities which can mutate 1263 // the region state. For example, a split procedure may already be running for this parent. 1264 // A split procedure cannot succeed if the parent region is no longer open, so we can 1265 // ignore it in that case. 1266 // Note that submitting more than one split procedure for a given region is 1267 // harmless -- the split is fenced in the procedure handling -- but it would be noisy in 1268 // the logs. Only one procedure can succeed. The other procedure(s) would abort during 1269 // initialization and report failure with WARN level logging. 1270 RegionState parentState = regionStates.getRegionState(parent); 1271 if (parentState != null && parentState.isOpened()) { 1272 master.getMasterProcedureExecutor().submitProcedure(createSplitProcedure(parent, splitKey)); 1273 } else { 1274 LOG.info("Ignoring split request from " + serverName + ", parent=" + parent 1275 + " because parent is unknown or not open"); 1276 return; 1277 } 1278 1279 // If the RS is < 2.0 throw an exception to abort the operation, we are handling the split 1280 if (master.getServerManager().getVersionNumber(serverName) < 0x0200000) { 1281 throw new UnsupportedOperationException( 1282 String.format("Split handled by the master: " + "parent=%s hriA=%s hriB=%s", 1283 parent.getShortNameToLog(), hriA, hriB)); 1284 } 1285 } 1286 1287 private void updateRegionMergeTransition(final ServerName serverName, final TransitionCode state, 1288 final RegionInfo merged, final RegionInfo hriA, final RegionInfo hriB) throws IOException { 1289 checkMetaLoaded(merged); 1290 1291 if (state != TransitionCode.READY_TO_MERGE) { 1292 throw new UnexpectedStateException( 1293 "Unsupported merge regionState=" + state + " for regionA=" + hriA + " regionB=" + hriB 1294 + " merged=" + merged + " maybe an old RS (< 2.0) had the operation in progress"); 1295 } 1296 1297 if (!master.isSplitOrMergeEnabled(MasterSwitchType.MERGE)) { 1298 LOG.warn("Merge switch is off! skip merge of regionA=" + hriA + " regionB=" + hriB); 1299 throw new DoNotRetryIOException( 1300 "Merge of regionA=" + hriA + " regionB=" + hriB + " failed because merge switch is off"); 1301 } 1302 1303 // Submit the Merge procedure 1304 if (LOG.isDebugEnabled()) { 1305 LOG.debug("Handling merge request from RS=" + merged + ", merged=" + merged); 1306 } 1307 master.getMasterProcedureExecutor().submitProcedure(createMergeProcedure(hriA, hriB)); 1308 1309 // If the RS is < 2.0 throw an exception to abort the operation, we are handling the merge 1310 if (master.getServerManager().getVersionNumber(serverName) < 0x0200000) { 1311 throw new UnsupportedOperationException( 1312 String.format("Merge not handled yet: regionState=%s merged=%s hriA=%s hriB=%s", state, 1313 merged, hriA, hriB)); 1314 } 1315 } 1316 1317 // ============================================================================================ 1318 // RS Status update (report online regions) helpers 1319 // ============================================================================================ 1320 /** 1321 * The master will call this method when the RS send the regionServerReport(). The report will 1322 * contains the "online regions". This method will check the the online regions against the 1323 * in-memory state of the AM, and we will log a warn message if there is a mismatch. This is 1324 * because that there is no fencing between the reportRegionStateTransition method and 1325 * regionServerReport method, so there could be race and introduce inconsistency here, but 1326 * actually there is no problem. 1327 * <p/> 1328 * Please see HBASE-21421 and HBASE-21463 for more details. 1329 */ 1330 public void reportOnlineRegions(ServerName serverName, Set<byte[]> regionNames) { 1331 if (!isRunning()) { 1332 return; 1333 } 1334 if (LOG.isTraceEnabled()) { 1335 LOG.trace("ReportOnlineRegions {} regionCount={}, metaLoaded={} {}", serverName, 1336 regionNames.size(), isMetaLoaded(), 1337 regionNames.stream().map(Bytes::toStringBinary).collect(Collectors.toList())); 1338 } 1339 1340 ServerStateNode serverNode = regionStates.getOrCreateServer(serverName); 1341 synchronized (serverNode) { 1342 if (!serverNode.isInState(ServerState.ONLINE)) { 1343 LOG.warn("Got a report from a server result in state " + serverNode.getState()); 1344 return; 1345 } 1346 } 1347 1348 // Track the regionserver reported online regions in memory. 1349 synchronized (rsReports) { 1350 rsReports.put(serverName, regionNames); 1351 } 1352 1353 if (regionNames.isEmpty()) { 1354 // nothing to do if we don't have regions 1355 LOG.trace("no online region found on {}", serverName); 1356 return; 1357 } 1358 if (!isMetaLoaded()) { 1359 // we are still on startup, skip checking 1360 return; 1361 } 1362 // The Heartbeat tells us of what regions are on the region serve, check the state. 1363 checkOnlineRegionsReport(serverNode, regionNames); 1364 } 1365 1366 /** 1367 * Close <code>regionName</code> on <code>sn</code> silently and immediately without using a 1368 * Procedure or going via hbase:meta. For case where a RegionServer's hosting of a Region is not 1369 * aligned w/ the Master's accounting of Region state. This is for cleaning up an error in 1370 * accounting. 1371 */ 1372 private void closeRegionSilently(ServerName sn, byte[] regionName) { 1373 try { 1374 RegionInfo ri = CatalogFamilyFormat.parseRegionInfoFromRegionName(regionName); 1375 // Pass -1 for timeout. Means do not wait. 1376 ServerManager.closeRegionSilentlyAndWait(this.master.getAsyncClusterConnection(), sn, ri, -1); 1377 } catch (Exception e) { 1378 LOG.error("Failed trying to close {} on {}", Bytes.toStringBinary(regionName), sn, e); 1379 } 1380 } 1381 1382 /** 1383 * Check that what the RegionServer reports aligns with the Master's image. If disagreement, we 1384 * will tell the RegionServer to expediently close a Region we do not think it should have. 1385 */ 1386 private void checkOnlineRegionsReport(ServerStateNode serverNode, Set<byte[]> regionNames) { 1387 ServerName serverName = serverNode.getServerName(); 1388 for (byte[] regionName : regionNames) { 1389 if (!isRunning()) { 1390 return; 1391 } 1392 RegionStateNode regionNode = regionStates.getRegionStateNodeFromName(regionName); 1393 if (regionNode == null) { 1394 String regionNameAsStr = Bytes.toStringBinary(regionName); 1395 LOG.warn("No RegionStateNode for {} but reported as up on {}; closing...", regionNameAsStr, 1396 serverName); 1397 closeRegionSilently(serverNode.getServerName(), regionName); 1398 continue; 1399 } 1400 final long lag = 1000; 1401 regionNode.lock(); 1402 try { 1403 long diff = EnvironmentEdgeManager.currentTime() - regionNode.getLastUpdate(); 1404 if (regionNode.isInState(State.OPENING, State.OPEN)) { 1405 // This is possible as a region server has just closed a region but the region server 1406 // report is generated before the closing, but arrive after the closing. Make sure there 1407 // is some elapsed time so less false alarms. 1408 if (!regionNode.getRegionLocation().equals(serverName) && diff > lag) { 1409 LOG.warn("Reporting {} server does not match {} (time since last " 1410 + "update={}ms); closing...", serverName, regionNode, diff); 1411 closeRegionSilently(serverNode.getServerName(), regionName); 1412 } 1413 } else if (!regionNode.isInState(State.CLOSING, State.SPLITTING)) { 1414 // So, we can get report that a region is CLOSED or SPLIT because a heartbeat 1415 // came in at about same time as a region transition. Make sure there is some 1416 // elapsed time so less false alarms. 1417 if (diff > lag) { 1418 LOG.warn("Reporting {} state does not match {} (time since last update={}ms)", 1419 serverName, regionNode, diff); 1420 } 1421 } 1422 } finally { 1423 regionNode.unlock(); 1424 } 1425 } 1426 } 1427 1428 // ============================================================================================ 1429 // RIT chore 1430 // ============================================================================================ 1431 private static class RegionInTransitionChore extends ProcedureInMemoryChore<MasterProcedureEnv> { 1432 public RegionInTransitionChore(final int timeoutMsec) { 1433 super(timeoutMsec); 1434 } 1435 1436 @Override 1437 protected void periodicExecute(final MasterProcedureEnv env) { 1438 final AssignmentManager am = env.getAssignmentManager(); 1439 1440 final RegionInTransitionStat ritStat = am.computeRegionInTransitionStat(); 1441 if (ritStat.hasRegionsOverThreshold()) { 1442 for (RegionState hri : ritStat.getRegionOverThreshold()) { 1443 am.handleRegionOverStuckWarningThreshold(hri.getRegion()); 1444 } 1445 } 1446 1447 // update metrics 1448 am.updateRegionsInTransitionMetrics(ritStat); 1449 } 1450 } 1451 1452 private static class DeadServerMetricRegionChore 1453 extends ProcedureInMemoryChore<MasterProcedureEnv> { 1454 public DeadServerMetricRegionChore(final int timeoutMsec) { 1455 super(timeoutMsec); 1456 } 1457 1458 @Override 1459 protected void periodicExecute(final MasterProcedureEnv env) { 1460 final ServerManager sm = env.getMasterServices().getServerManager(); 1461 final AssignmentManager am = env.getAssignmentManager(); 1462 // To minimize inconsistencies we are not going to snapshot live servers in advance in case 1463 // new servers are added; OTOH we don't want to add heavy sync for a consistent view since 1464 // this is for metrics. Instead, we're going to check each regions as we go; to avoid making 1465 // too many checks, we maintain a local lists of server, limiting us to false negatives. If 1466 // we miss some recently-dead server, we'll just see it next time. 1467 Set<ServerName> recentlyLiveServers = new HashSet<>(); 1468 int deadRegions = 0, unknownRegions = 0; 1469 for (RegionStateNode rsn : am.getRegionStates().getRegionStateNodes()) { 1470 if (rsn.getState() != State.OPEN) { 1471 continue; // Opportunistic check, should quickly skip RITs, offline tables, etc. 1472 } 1473 // Do not need to acquire region state lock as this is only for showing metrics. 1474 ServerName sn = rsn.getRegionLocation(); 1475 State state = rsn.getState(); 1476 if (state != State.OPEN) { 1477 continue; // Mostly skipping RITs that are already being take care of. 1478 } 1479 if (sn == null) { 1480 ++unknownRegions; // Opened on null? 1481 continue; 1482 } 1483 if (recentlyLiveServers.contains(sn)) { 1484 continue; 1485 } 1486 ServerManager.ServerLiveState sls = sm.isServerKnownAndOnline(sn); 1487 switch (sls) { 1488 case LIVE: 1489 recentlyLiveServers.add(sn); 1490 break; 1491 case DEAD: 1492 ++deadRegions; 1493 break; 1494 case UNKNOWN: 1495 ++unknownRegions; 1496 break; 1497 default: 1498 throw new AssertionError("Unexpected " + sls); 1499 } 1500 } 1501 if (deadRegions > 0 || unknownRegions > 0) { 1502 LOG.info("Found {} OPEN regions on dead servers and {} OPEN regions on unknown servers", 1503 deadRegions, unknownRegions); 1504 } 1505 1506 am.updateDeadServerRegionMetrics(deadRegions, unknownRegions); 1507 } 1508 } 1509 1510 public RegionInTransitionStat computeRegionInTransitionStat() { 1511 final RegionInTransitionStat rit = new RegionInTransitionStat(getConfiguration()); 1512 rit.update(this); 1513 return rit; 1514 } 1515 1516 public static class RegionInTransitionStat { 1517 private final int ritThreshold; 1518 1519 private HashMap<String, RegionState> ritsOverThreshold = null; 1520 private long statTimestamp; 1521 private long oldestRITTime = 0; 1522 private int totalRITsTwiceThreshold = 0; 1523 private int totalRITs = 0; 1524 1525 public RegionInTransitionStat(final Configuration conf) { 1526 this.ritThreshold = 1527 conf.getInt(METRICS_RIT_STUCK_WARNING_THRESHOLD, DEFAULT_RIT_STUCK_WARNING_THRESHOLD); 1528 } 1529 1530 public int getRITThreshold() { 1531 return ritThreshold; 1532 } 1533 1534 public long getTimestamp() { 1535 return statTimestamp; 1536 } 1537 1538 public int getTotalRITs() { 1539 return totalRITs; 1540 } 1541 1542 public long getOldestRITTime() { 1543 return oldestRITTime; 1544 } 1545 1546 public int getTotalRITsOverThreshold() { 1547 Map<String, RegionState> m = this.ritsOverThreshold; 1548 return m != null ? m.size() : 0; 1549 } 1550 1551 public boolean hasRegionsTwiceOverThreshold() { 1552 return totalRITsTwiceThreshold > 0; 1553 } 1554 1555 public boolean hasRegionsOverThreshold() { 1556 Map<String, RegionState> m = this.ritsOverThreshold; 1557 return m != null && !m.isEmpty(); 1558 } 1559 1560 public Collection<RegionState> getRegionOverThreshold() { 1561 Map<String, RegionState> m = this.ritsOverThreshold; 1562 return m != null ? m.values() : Collections.emptySet(); 1563 } 1564 1565 public boolean isRegionOverThreshold(final RegionInfo regionInfo) { 1566 Map<String, RegionState> m = this.ritsOverThreshold; 1567 return m != null && m.containsKey(regionInfo.getEncodedName()); 1568 } 1569 1570 public boolean isRegionTwiceOverThreshold(final RegionInfo regionInfo) { 1571 Map<String, RegionState> m = this.ritsOverThreshold; 1572 if (m == null) { 1573 return false; 1574 } 1575 final RegionState state = m.get(regionInfo.getEncodedName()); 1576 if (state == null) { 1577 return false; 1578 } 1579 return (statTimestamp - state.getStamp()) > (ritThreshold * 2); 1580 } 1581 1582 protected void update(final AssignmentManager am) { 1583 final RegionStates regionStates = am.getRegionStates(); 1584 this.statTimestamp = EnvironmentEdgeManager.currentTime(); 1585 update(regionStates.getRegionsStateInTransition(), statTimestamp); 1586 update(regionStates.getRegionFailedOpen(), statTimestamp); 1587 1588 if (LOG.isDebugEnabled() && ritsOverThreshold != null && !ritsOverThreshold.isEmpty()) { 1589 LOG.debug("RITs over threshold: {}", 1590 ritsOverThreshold.entrySet().stream() 1591 .map(e -> e.getKey() + ":" + e.getValue().getState().name()) 1592 .collect(Collectors.joining("\n"))); 1593 } 1594 } 1595 1596 private void update(final Collection<RegionState> regions, final long currentTime) { 1597 for (RegionState state : regions) { 1598 totalRITs++; 1599 final long ritStartedMs = state.getStamp(); 1600 if (ritStartedMs == 0) { 1601 // Don't output bogus values to metrics if they accidentally make it here. 1602 LOG.warn("The RIT {} has no start time", state.getRegion()); 1603 continue; 1604 } 1605 final long ritTime = currentTime - ritStartedMs; 1606 if (ritTime > ritThreshold) { 1607 if (ritsOverThreshold == null) { 1608 ritsOverThreshold = new HashMap<String, RegionState>(); 1609 } 1610 ritsOverThreshold.put(state.getRegion().getEncodedName(), state); 1611 totalRITsTwiceThreshold += (ritTime > (ritThreshold * 2)) ? 1 : 0; 1612 } 1613 if (oldestRITTime < ritTime) { 1614 oldestRITTime = ritTime; 1615 } 1616 } 1617 } 1618 } 1619 1620 private void updateRegionsInTransitionMetrics(final RegionInTransitionStat ritStat) { 1621 metrics.updateRITOldestAge(ritStat.getOldestRITTime()); 1622 metrics.updateRITCount(ritStat.getTotalRITs()); 1623 metrics.updateRITCountOverThreshold(ritStat.getTotalRITsOverThreshold()); 1624 } 1625 1626 private void updateDeadServerRegionMetrics(int deadRegions, int unknownRegions) { 1627 metrics.updateDeadServerOpenRegions(deadRegions); 1628 metrics.updateUnknownServerOpenRegions(unknownRegions); 1629 } 1630 1631 private void handleRegionOverStuckWarningThreshold(final RegionInfo regionInfo) { 1632 final RegionStateNode regionNode = regionStates.getRegionStateNode(regionInfo); 1633 // if (regionNode.isStuck()) { 1634 LOG.warn("STUCK Region-In-Transition {}", regionNode); 1635 } 1636 1637 // ============================================================================================ 1638 // TODO: Master load/bootstrap 1639 // ============================================================================================ 1640 public void joinCluster() throws IOException { 1641 long startTime = System.nanoTime(); 1642 LOG.debug("Joining cluster..."); 1643 1644 // Scan hbase:meta to build list of existing regions, servers, and assignment. 1645 // hbase:meta is online now or will be. Inside loadMeta, we keep trying. Can't make progress 1646 // w/o meta. 1647 loadMeta(); 1648 1649 while (master.getServerManager().countOfRegionServers() < 1) { 1650 LOG.info("Waiting for RegionServers to join; current count={}", 1651 master.getServerManager().countOfRegionServers()); 1652 Threads.sleep(250); 1653 } 1654 LOG.info("Number of RegionServers={}", master.getServerManager().countOfRegionServers()); 1655 1656 // Start the chores 1657 master.getMasterProcedureExecutor().addChore(this.ritChore); 1658 master.getMasterProcedureExecutor().addChore(this.deadMetricChore); 1659 1660 long costMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); 1661 LOG.info("Joined the cluster in {}", StringUtils.humanTimeDiff(costMs)); 1662 } 1663 1664 /** 1665 * Create assign procedure for offline regions. Just follow the old 1666 * processofflineServersWithOnlineRegions method. Since now we do not need to deal with dead 1667 * server any more, we only deal with the regions in OFFLINE state in this method. And this is a 1668 * bit strange, that for new regions, we will add it in CLOSED state instead of OFFLINE state, and 1669 * usually there will be a procedure to track them. The processofflineServersWithOnlineRegions is 1670 * a legacy from long ago, as things are going really different now, maybe we do not need this 1671 * method any more. Need to revisit later. 1672 */ 1673 // Public so can be run by the Master as part of the startup. Needs hbase:meta to be online. 1674 // Needs to be done after the table state manager has been started. 1675 public void processOfflineRegions() { 1676 TransitRegionStateProcedure[] procs = 1677 regionStates.getRegionStateNodes().stream().filter(rsn -> rsn.isInState(State.OFFLINE)) 1678 .filter(rsn -> isTableEnabled(rsn.getRegionInfo().getTable())).map(rsn -> { 1679 rsn.lock(); 1680 try { 1681 if (rsn.getProcedure() != null) { 1682 return null; 1683 } else { 1684 return rsn.setProcedure(TransitRegionStateProcedure.assign(getProcedureEnvironment(), 1685 rsn.getRegionInfo(), null)); 1686 } 1687 } finally { 1688 rsn.unlock(); 1689 } 1690 }).filter(p -> p != null).toArray(TransitRegionStateProcedure[]::new); 1691 if (procs.length > 0) { 1692 master.getMasterProcedureExecutor().submitProcedures(procs); 1693 } 1694 } 1695 1696 /* 1697 * AM internal RegionStateStore.RegionStateVisitor implementation. To be used when scanning META 1698 * table for region rows, using RegionStateStore utility methods. RegionStateStore methods will 1699 * convert Result into proper RegionInfo instances, but those would still need to be added into 1700 * AssignmentManager.regionStates in-memory cache. RegionMetaLoadingVisitor.visitRegionState 1701 * method provides the logic for adding RegionInfo instances as loaded from latest META scan into 1702 * AssignmentManager.regionStates. 1703 */ 1704 private class RegionMetaLoadingVisitor implements RegionStateStore.RegionStateVisitor { 1705 1706 @Override 1707 public void visitRegionState(Result result, final RegionInfo regionInfo, final State state, 1708 final ServerName regionLocation, final ServerName lastHost, final long openSeqNum) { 1709 if ( 1710 state == null && regionLocation == null && lastHost == null 1711 && openSeqNum == SequenceId.NO_SEQUENCE_ID 1712 ) { 1713 // This is a row with nothing in it. 1714 LOG.warn("Skipping empty row={}", result); 1715 return; 1716 } 1717 State localState = state; 1718 if (localState == null) { 1719 // No region state column data in hbase:meta table! Are I doing a rolling upgrade from 1720 // hbase1 to hbase2? Am I restoring a SNAPSHOT or otherwise adding a region to hbase:meta? 1721 // In any of these cases, state is empty. For now, presume OFFLINE but there are probably 1722 // cases where we need to probe more to be sure this correct; TODO informed by experience. 1723 LOG.info(regionInfo.getEncodedName() + " regionState=null; presuming " + State.OFFLINE); 1724 localState = State.OFFLINE; 1725 } 1726 RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo); 1727 // Do not need to lock on regionNode, as we can make sure that before we finish loading 1728 // meta, all the related procedures can not be executed. The only exception is for meta 1729 // region related operations, but here we do not load the informations for meta region. 1730 regionNode.setState(localState); 1731 regionNode.setLastHost(lastHost); 1732 regionNode.setRegionLocation(regionLocation); 1733 regionNode.setOpenSeqNum(openSeqNum); 1734 1735 // Note: keep consistent with other methods, see region(Opening|Opened|Closing) 1736 // RIT/ServerCrash handling should take care of the transiting regions. 1737 if ( 1738 localState.matches(State.OPEN, State.OPENING, State.CLOSING, State.SPLITTING, State.MERGING) 1739 ) { 1740 assert regionLocation != null : "found null region location for " + regionNode; 1741 regionStates.addRegionToServer(regionNode); 1742 } else if (localState == State.OFFLINE || regionInfo.isOffline()) { 1743 regionStates.addToOfflineRegions(regionNode); 1744 } 1745 if (regionNode.getProcedure() != null) { 1746 regionNode.getProcedure().stateLoaded(AssignmentManager.this, regionNode); 1747 } 1748 } 1749 }; 1750 1751 /** 1752 * Attempt to load {@code regionInfo} from META, adding any results to the 1753 * {@link #regionStateStore} Is NOT aware of replica regions. 1754 * @param regionInfo the region to be loaded from META. 1755 * @throws IOException If some error occurs while querying META or parsing results. 1756 */ 1757 public void populateRegionStatesFromMeta(@NonNull final RegionInfo regionInfo) 1758 throws IOException { 1759 final String regionEncodedName = RegionInfo.DEFAULT_REPLICA_ID == regionInfo.getReplicaId() 1760 ? regionInfo.getEncodedName() 1761 : RegionInfoBuilder.newBuilder(regionInfo).setReplicaId(RegionInfo.DEFAULT_REPLICA_ID).build() 1762 .getEncodedName(); 1763 populateRegionStatesFromMeta(regionEncodedName); 1764 } 1765 1766 /** 1767 * Attempt to load {@code regionEncodedName} from META, adding any results to the 1768 * {@link #regionStateStore} Is NOT aware of replica regions. 1769 * @param regionEncodedName encoded name for the region to be loaded from META. 1770 * @throws IOException If some error occurs while querying META or parsing results. 1771 */ 1772 public void populateRegionStatesFromMeta(@NonNull String regionEncodedName) throws IOException { 1773 final RegionMetaLoadingVisitor visitor = new RegionMetaLoadingVisitor(); 1774 regionStateStore.visitMetaForRegion(regionEncodedName, visitor); 1775 } 1776 1777 private void loadMeta() throws IOException { 1778 // TODO: use a thread pool 1779 regionStateStore.visitMeta(new RegionMetaLoadingVisitor()); 1780 } 1781 1782 /** 1783 * Used to check if the meta loading is done. 1784 * <p/> 1785 * if not we throw PleaseHoldException since we are rebuilding the RegionStates 1786 * @param hri region to check if it is already rebuild 1787 * @throws PleaseHoldException if meta has not been loaded yet 1788 */ 1789 private void checkMetaLoaded(RegionInfo hri) throws PleaseHoldException { 1790 if (!isRunning()) { 1791 throw new PleaseHoldException("AssignmentManager not running"); 1792 } 1793 boolean meta = isMetaRegion(hri); 1794 boolean metaLoaded = isMetaLoaded(); 1795 if (!meta && !metaLoaded) { 1796 throw new PleaseHoldException( 1797 "Master not fully online; hbase:meta=" + meta + ", metaLoaded=" + metaLoaded); 1798 } 1799 } 1800 1801 // ============================================================================================ 1802 // TODO: Metrics 1803 // ============================================================================================ 1804 public int getNumRegionsOpened() { 1805 // TODO: Used by TestRegionPlacement.java and assume monotonically increasing value 1806 return 0; 1807 } 1808 1809 /** 1810 * Usually run by the Master in reaction to server crash during normal processing. Can also be 1811 * invoked via external RPC to effect repair; in the latter case, the 'force' flag is set so we 1812 * push through the SCP though context may indicate already-running-SCP (An old SCP may have 1813 * exited abnormally, or damaged cluster may still have references in hbase:meta to 'Unknown 1814 * Servers' -- servers that are not online or in dead servers list, etc.) 1815 * @param force Set if the request came in externally over RPC (via hbck2). Force means run the 1816 * SCP even if it seems as though there might be an outstanding SCP running. 1817 * @return pid of scheduled SCP or {@link Procedure#NO_PROC_ID} if none scheduled. 1818 */ 1819 public long submitServerCrash(ServerName serverName, boolean shouldSplitWal, boolean force) { 1820 // May be an 'Unknown Server' so handle case where serverNode is null. 1821 ServerStateNode serverNode = regionStates.getServerNode(serverName); 1822 // Remove the in-memory rsReports result 1823 synchronized (rsReports) { 1824 rsReports.remove(serverName); 1825 } 1826 1827 // We hold the write lock here for fencing on reportRegionStateTransition. Once we set the 1828 // server state to CRASHED, we will no longer accept the reportRegionStateTransition call from 1829 // this server. This is used to simplify the implementation for TRSP and SCP, where we can make 1830 // sure that, the region list fetched by SCP will not be changed any more. 1831 if (serverNode != null) { 1832 serverNode.writeLock().lock(); 1833 } 1834 boolean carryingMeta; 1835 long pid; 1836 try { 1837 ProcedureExecutor<MasterProcedureEnv> procExec = this.master.getMasterProcedureExecutor(); 1838 carryingMeta = isCarryingMeta(serverName); 1839 if (!force && serverNode != null && !serverNode.isInState(ServerState.ONLINE)) { 1840 LOG.info("Skip adding ServerCrashProcedure for {} (meta={}) -- running?", serverNode, 1841 carryingMeta); 1842 return Procedure.NO_PROC_ID; 1843 } else { 1844 MasterProcedureEnv mpe = procExec.getEnvironment(); 1845 // If serverNode == null, then 'Unknown Server'. Schedule HBCKSCP instead. 1846 // HBCKSCP scours Master in-memory state AND hbase;meta for references to 1847 // serverName just-in-case. An SCP that is scheduled when the server is 1848 // 'Unknown' probably originated externally with HBCK2 fix-it tool. 1849 ServerState oldState = null; 1850 if (serverNode != null) { 1851 oldState = serverNode.getState(); 1852 serverNode.setState(ServerState.CRASHED); 1853 } 1854 1855 if (force) { 1856 pid = procExec.submitProcedure( 1857 new HBCKServerCrashProcedure(mpe, serverName, shouldSplitWal, carryingMeta)); 1858 } else { 1859 pid = procExec.submitProcedure( 1860 new ServerCrashProcedure(mpe, serverName, shouldSplitWal, carryingMeta)); 1861 } 1862 LOG.info("Scheduled ServerCrashProcedure pid={} for {} (carryingMeta={}){}.", pid, 1863 serverName, carryingMeta, 1864 serverNode == null ? "" : " " + serverNode.toString() + ", oldState=" + oldState); 1865 } 1866 } finally { 1867 if (serverNode != null) { 1868 serverNode.writeLock().unlock(); 1869 } 1870 } 1871 return pid; 1872 } 1873 1874 public void offlineRegion(final RegionInfo regionInfo) { 1875 // TODO used by MasterRpcServices 1876 RegionStateNode node = regionStates.getRegionStateNode(regionInfo); 1877 if (node != null) { 1878 node.offline(); 1879 } 1880 } 1881 1882 public void onlineRegion(final RegionInfo regionInfo, final ServerName serverName) { 1883 // TODO used by TestSplitTransactionOnCluster.java 1884 } 1885 1886 public Map<ServerName, List<RegionInfo>> 1887 getSnapShotOfAssignment(final Collection<RegionInfo> regions) { 1888 return regionStates.getSnapShotOfAssignment(regions); 1889 } 1890 1891 // ============================================================================================ 1892 // TODO: UTILS/HELPERS? 1893 // ============================================================================================ 1894 /** 1895 * Used by the client (via master) to identify if all regions have the schema updates 1896 * @return Pair indicating the status of the alter command (pending/total) 1897 */ 1898 public Pair<Integer, Integer> getReopenStatus(TableName tableName) { 1899 if (isTableDisabled(tableName)) { 1900 return new Pair<Integer, Integer>(0, 0); 1901 } 1902 1903 final List<RegionState> states = regionStates.getTableRegionStates(tableName); 1904 int ritCount = 0; 1905 for (RegionState regionState : states) { 1906 if (!regionState.isOpened() && !regionState.isSplit()) { 1907 ritCount++; 1908 } 1909 } 1910 return new Pair<Integer, Integer>(ritCount, states.size()); 1911 } 1912 1913 // ============================================================================================ 1914 // TODO: Region State In Transition 1915 // ============================================================================================ 1916 public boolean hasRegionsInTransition() { 1917 return regionStates.hasRegionsInTransition(); 1918 } 1919 1920 public List<RegionStateNode> getRegionsInTransition() { 1921 return regionStates.getRegionsInTransition(); 1922 } 1923 1924 public List<RegionInfo> getAssignedRegions() { 1925 return regionStates.getAssignedRegions(); 1926 } 1927 1928 /** 1929 * Resolve a cached {@link RegionInfo} from the region name as a {@code byte[]}. 1930 */ 1931 public RegionInfo getRegionInfo(final byte[] regionName) { 1932 final RegionStateNode regionState = regionStates.getRegionStateNodeFromName(regionName); 1933 return regionState != null ? regionState.getRegionInfo() : null; 1934 } 1935 1936 /** 1937 * Resolve a cached {@link RegionInfo} from the encoded region name as a {@code String}. 1938 */ 1939 public RegionInfo getRegionInfo(final String encodedRegionName) { 1940 final RegionStateNode regionState = 1941 regionStates.getRegionStateNodeFromEncodedRegionName(encodedRegionName); 1942 return regionState != null ? regionState.getRegionInfo() : null; 1943 } 1944 1945 // ============================================================================================ 1946 // Expected states on region state transition. 1947 // Notice that there is expected states for transiting to OPENING state, this is because SCP. 1948 // See the comments in regionOpening method for more details. 1949 // ============================================================================================ 1950 private static final State[] STATES_EXPECTED_ON_OPEN = { State.OPENING, // Normal case 1951 State.OPEN // Retrying 1952 }; 1953 1954 private static final State[] STATES_EXPECTED_ON_CLOSING = { State.OPEN, // Normal case 1955 State.CLOSING, // Retrying 1956 State.SPLITTING, // Offline the split parent 1957 State.MERGING // Offline the merge parents 1958 }; 1959 1960 private static final State[] STATES_EXPECTED_ON_CLOSED = { State.CLOSING, // Normal case 1961 State.CLOSED // Retrying 1962 }; 1963 1964 // This is for manually scheduled region assign, can add other states later if we find out other 1965 // usages 1966 private static final State[] STATES_EXPECTED_ON_ASSIGN = { State.CLOSED, State.OFFLINE }; 1967 1968 // We only allow unassign or move a region which is in OPEN state. 1969 private static final State[] STATES_EXPECTED_ON_UNASSIGN_OR_MOVE = { State.OPEN }; 1970 1971 // ============================================================================================ 1972 // Region Status update 1973 // Should only be called in TransitRegionStateProcedure(and related procedures), as the locking 1974 // and pre-assumptions are very tricky. 1975 // ============================================================================================ 1976 private void transitStateAndUpdate(RegionStateNode regionNode, RegionState.State newState, 1977 RegionState.State... expectedStates) throws IOException { 1978 RegionState.State state = regionNode.getState(); 1979 regionNode.transitionState(newState, expectedStates); 1980 boolean succ = false; 1981 try { 1982 regionStateStore.updateRegionLocation(regionNode); 1983 succ = true; 1984 } finally { 1985 if (!succ) { 1986 // revert 1987 regionNode.setState(state); 1988 } 1989 } 1990 } 1991 1992 // should be called within the synchronized block of RegionStateNode 1993 void regionOpening(RegionStateNode regionNode) throws IOException { 1994 // As in SCP, for performance reason, there is no TRSP attached with this region, we will not 1995 // update the region state, which means that the region could be in any state when we want to 1996 // assign it after a RS crash. So here we do not pass the expectedStates parameter. 1997 transitStateAndUpdate(regionNode, State.OPENING); 1998 regionStates.addRegionToServer(regionNode); 1999 // update the operation count metrics 2000 metrics.incrementOperationCounter(); 2001 } 2002 2003 // should be called under the RegionStateNode lock 2004 // The parameter 'giveUp' means whether we will try to open the region again, if it is true, then 2005 // we will persist the FAILED_OPEN state into hbase:meta. 2006 void regionFailedOpen(RegionStateNode regionNode, boolean giveUp) throws IOException { 2007 RegionState.State state = regionNode.getState(); 2008 ServerName regionLocation = regionNode.getRegionLocation(); 2009 if (giveUp) { 2010 regionNode.setState(State.FAILED_OPEN); 2011 regionNode.setRegionLocation(null); 2012 boolean succ = false; 2013 try { 2014 regionStateStore.updateRegionLocation(regionNode); 2015 succ = true; 2016 } finally { 2017 if (!succ) { 2018 // revert 2019 regionNode.setState(state); 2020 regionNode.setRegionLocation(regionLocation); 2021 } 2022 } 2023 } 2024 if (regionLocation != null) { 2025 regionStates.removeRegionFromServer(regionLocation, regionNode); 2026 } 2027 } 2028 2029 // should be called under the RegionStateNode lock 2030 void regionClosing(RegionStateNode regionNode) throws IOException { 2031 transitStateAndUpdate(regionNode, State.CLOSING, STATES_EXPECTED_ON_CLOSING); 2032 2033 RegionInfo hri = regionNode.getRegionInfo(); 2034 // Set meta has not initialized early. so people trying to create/edit tables will wait 2035 if (isMetaRegion(hri)) { 2036 setMetaAssigned(hri, false); 2037 } 2038 regionStates.addRegionToServer(regionNode); 2039 // update the operation count metrics 2040 metrics.incrementOperationCounter(); 2041 } 2042 2043 // for open and close, they will first be persist to the procedure store in 2044 // RegionRemoteProcedureBase. So here we will first change the in memory state as it is considered 2045 // as succeeded if the persistence to procedure store is succeeded, and then when the 2046 // RegionRemoteProcedureBase is woken up, we will persist the RegionStateNode to hbase:meta. 2047 2048 // should be called under the RegionStateNode lock 2049 void regionOpenedWithoutPersistingToMeta(RegionStateNode regionNode) throws IOException { 2050 regionNode.transitionState(State.OPEN, STATES_EXPECTED_ON_OPEN); 2051 RegionInfo regionInfo = regionNode.getRegionInfo(); 2052 regionStates.addRegionToServer(regionNode); 2053 regionStates.removeFromFailedOpen(regionInfo); 2054 } 2055 2056 // should be called under the RegionStateNode lock 2057 void regionClosedWithoutPersistingToMeta(RegionStateNode regionNode) throws IOException { 2058 ServerName regionLocation = regionNode.getRegionLocation(); 2059 regionNode.transitionState(State.CLOSED, STATES_EXPECTED_ON_CLOSED); 2060 regionNode.setRegionLocation(null); 2061 if (regionLocation != null) { 2062 regionNode.setLastHost(regionLocation); 2063 regionStates.removeRegionFromServer(regionLocation, regionNode); 2064 } 2065 } 2066 2067 // should be called under the RegionStateNode lock 2068 // for SCP 2069 public void regionClosedAbnormally(RegionStateNode regionNode) throws IOException { 2070 RegionState.State state = regionNode.getState(); 2071 ServerName regionLocation = regionNode.getRegionLocation(); 2072 regionNode.transitionState(State.ABNORMALLY_CLOSED); 2073 regionNode.setRegionLocation(null); 2074 boolean succ = false; 2075 try { 2076 regionStateStore.updateRegionLocation(regionNode); 2077 succ = true; 2078 } finally { 2079 if (!succ) { 2080 // revert 2081 regionNode.setState(state); 2082 regionNode.setRegionLocation(regionLocation); 2083 } 2084 } 2085 if (regionLocation != null) { 2086 regionNode.setLastHost(regionLocation); 2087 regionStates.removeRegionFromServer(regionLocation, regionNode); 2088 } 2089 } 2090 2091 void persistToMeta(RegionStateNode regionNode) throws IOException { 2092 regionStateStore.updateRegionLocation(regionNode); 2093 RegionInfo regionInfo = regionNode.getRegionInfo(); 2094 if (isMetaRegion(regionInfo) && regionNode.getState() == State.OPEN) { 2095 // Usually we'd set a table ENABLED at this stage but hbase:meta is ALWAYs enabled, it 2096 // can't be disabled -- so skip the RPC (besides... enabled is managed by TableStateManager 2097 // which is backed by hbase:meta... Avoid setting ENABLED to avoid having to update state 2098 // on table that contains state. 2099 setMetaAssigned(regionInfo, true); 2100 } 2101 } 2102 2103 // ============================================================================================ 2104 // The above methods can only be called in TransitRegionStateProcedure(and related procedures) 2105 // ============================================================================================ 2106 2107 public void markRegionAsSplit(final RegionInfo parent, final ServerName serverName, 2108 final RegionInfo daughterA, final RegionInfo daughterB) throws IOException { 2109 // Update hbase:meta. Parent will be marked offline and split up in hbase:meta. 2110 // The parent stays in regionStates until cleared when removed by CatalogJanitor. 2111 // Update its state in regionStates to it shows as offline and split when read 2112 // later figuring what regions are in a table and what are not: see 2113 // regionStates#getRegionsOfTable 2114 final RegionStateNode node = regionStates.getOrCreateRegionStateNode(parent); 2115 node.setState(State.SPLIT); 2116 final RegionStateNode nodeA = regionStates.getOrCreateRegionStateNode(daughterA); 2117 nodeA.setState(State.SPLITTING_NEW); 2118 final RegionStateNode nodeB = regionStates.getOrCreateRegionStateNode(daughterB); 2119 nodeB.setState(State.SPLITTING_NEW); 2120 2121 TableDescriptor td = master.getTableDescriptors().get(parent.getTable()); 2122 // TODO: here we just update the parent region info in meta, to set split and offline to true, 2123 // without changing the one in the region node. This is a bit confusing but the region info 2124 // field in RegionStateNode is not expected to be changed in the current design. Need to find a 2125 // possible way to address this problem, or at least adding more comments about the trick to 2126 // deal with this problem, that when you want to filter out split parent, you need to check both 2127 // the RegionState on whether it is split, and also the region info. If one of them matches then 2128 // it is a split parent. And usually only one of them can match, as after restart, the region 2129 // state will be changed from SPLIT to CLOSED. 2130 regionStateStore.splitRegion(parent, daughterA, daughterB, serverName, td); 2131 if (shouldAssignFavoredNodes(parent)) { 2132 List<ServerName> onlineServers = this.master.getServerManager().getOnlineServersList(); 2133 getFavoredNodePromoter().generateFavoredNodesForDaughter(onlineServers, parent, daughterA, 2134 daughterB); 2135 } 2136 } 2137 2138 /** 2139 * When called here, the merge has happened. The merged regions have been unassigned and the above 2140 * markRegionClosed has been called on each so they have been disassociated from a hosting Server. 2141 * The merged region will be open after this call. The merged regions are removed from hbase:meta 2142 * below. Later they are deleted from the filesystem by the catalog janitor running against 2143 * hbase:meta. It notices when the merged region no longer holds references to the old regions 2144 * (References are deleted after a compaction rewrites what the Reference points at but not until 2145 * the archiver chore runs, are the References removed). 2146 */ 2147 public void markRegionAsMerged(final RegionInfo child, final ServerName serverName, 2148 RegionInfo[] mergeParents) throws IOException { 2149 final RegionStateNode node = regionStates.getOrCreateRegionStateNode(child); 2150 node.setState(State.MERGED); 2151 for (RegionInfo ri : mergeParents) { 2152 regionStates.deleteRegion(ri); 2153 } 2154 TableDescriptor td = master.getTableDescriptors().get(child.getTable()); 2155 regionStateStore.mergeRegions(child, mergeParents, serverName, td); 2156 if (shouldAssignFavoredNodes(child)) { 2157 getFavoredNodePromoter().generateFavoredNodesForMergedRegion(child, mergeParents); 2158 } 2159 } 2160 2161 /* 2162 * Favored nodes should be applied only when FavoredNodes balancer is configured and the region 2163 * belongs to a non-system table. 2164 */ 2165 private boolean shouldAssignFavoredNodes(RegionInfo region) { 2166 return this.shouldAssignRegionsWithFavoredNodes 2167 && FavoredNodesManager.isFavoredNodeApplicable(region); 2168 } 2169 2170 // ============================================================================================ 2171 // Assign Queue (Assign/Balance) 2172 // ============================================================================================ 2173 private final ArrayList<RegionStateNode> pendingAssignQueue = new ArrayList<RegionStateNode>(); 2174 private final ReentrantLock assignQueueLock = new ReentrantLock(); 2175 private final Condition assignQueueFullCond = assignQueueLock.newCondition(); 2176 2177 /** 2178 * Add the assign operation to the assignment queue. The pending assignment operation will be 2179 * processed, and each region will be assigned by a server using the balancer. 2180 */ 2181 protected void queueAssign(final RegionStateNode regionNode) { 2182 regionNode.getProcedureEvent().suspend(); 2183 2184 // TODO: quick-start for meta and the other sys-tables? 2185 assignQueueLock.lock(); 2186 try { 2187 pendingAssignQueue.add(regionNode); 2188 if ( 2189 regionNode.isSystemTable() || pendingAssignQueue.size() == 1 2190 || pendingAssignQueue.size() >= assignDispatchWaitQueueMaxSize 2191 ) { 2192 assignQueueFullCond.signal(); 2193 } 2194 } finally { 2195 assignQueueLock.unlock(); 2196 } 2197 } 2198 2199 private void startAssignmentThread() { 2200 assignThread = new Thread(master.getServerName().toShortString()) { 2201 @Override 2202 public void run() { 2203 while (isRunning()) { 2204 processAssignQueue(); 2205 } 2206 pendingAssignQueue.clear(); 2207 } 2208 }; 2209 assignThread.setDaemon(true); 2210 assignThread.start(); 2211 } 2212 2213 private void stopAssignmentThread() { 2214 assignQueueSignal(); 2215 try { 2216 while (assignThread.isAlive()) { 2217 assignQueueSignal(); 2218 assignThread.join(250); 2219 } 2220 } catch (InterruptedException e) { 2221 LOG.warn("join interrupted", e); 2222 Thread.currentThread().interrupt(); 2223 } 2224 } 2225 2226 private void assignQueueSignal() { 2227 assignQueueLock.lock(); 2228 try { 2229 assignQueueFullCond.signal(); 2230 } finally { 2231 assignQueueLock.unlock(); 2232 } 2233 } 2234 2235 @edu.umd.cs.findbugs.annotations.SuppressWarnings("WA_AWAIT_NOT_IN_LOOP") 2236 private HashMap<RegionInfo, RegionStateNode> waitOnAssignQueue() { 2237 HashMap<RegionInfo, RegionStateNode> regions = null; 2238 2239 assignQueueLock.lock(); 2240 try { 2241 if (pendingAssignQueue.isEmpty() && isRunning()) { 2242 assignQueueFullCond.await(); 2243 } 2244 2245 if (!isRunning()) { 2246 return null; 2247 } 2248 assignQueueFullCond.await(assignDispatchWaitMillis, TimeUnit.MILLISECONDS); 2249 regions = new HashMap<RegionInfo, RegionStateNode>(pendingAssignQueue.size()); 2250 for (RegionStateNode regionNode : pendingAssignQueue) { 2251 regions.put(regionNode.getRegionInfo(), regionNode); 2252 } 2253 pendingAssignQueue.clear(); 2254 } catch (InterruptedException e) { 2255 LOG.warn("got interrupted ", e); 2256 Thread.currentThread().interrupt(); 2257 } finally { 2258 assignQueueLock.unlock(); 2259 } 2260 return regions; 2261 } 2262 2263 private void processAssignQueue() { 2264 final HashMap<RegionInfo, RegionStateNode> regions = waitOnAssignQueue(); 2265 if (regions == null || regions.size() == 0 || !isRunning()) { 2266 return; 2267 } 2268 2269 if (LOG.isTraceEnabled()) { 2270 LOG.trace("PROCESS ASSIGN QUEUE regionCount=" + regions.size()); 2271 } 2272 2273 // TODO: Optimize balancer. pass a RegionPlan? 2274 final HashMap<RegionInfo, ServerName> retainMap = new HashMap<>(); 2275 final List<RegionInfo> userHRIs = new ArrayList<>(regions.size()); 2276 // Regions for system tables requiring reassignment 2277 final List<RegionInfo> systemHRIs = new ArrayList<>(); 2278 for (RegionStateNode regionStateNode : regions.values()) { 2279 boolean sysTable = regionStateNode.isSystemTable(); 2280 final List<RegionInfo> hris = sysTable ? systemHRIs : userHRIs; 2281 if (regionStateNode.getRegionLocation() != null) { 2282 retainMap.put(regionStateNode.getRegionInfo(), regionStateNode.getRegionLocation()); 2283 } else { 2284 hris.add(regionStateNode.getRegionInfo()); 2285 } 2286 } 2287 2288 // TODO: connect with the listener to invalidate the cache 2289 2290 // TODO use events 2291 List<ServerName> servers = master.getServerManager().createDestinationServersList(); 2292 for (int i = 0; servers.size() < 1; ++i) { 2293 // Report every fourth time around this loop; try not to flood log. 2294 if (i % 4 == 0) { 2295 LOG.warn("No servers available; cannot place " + regions.size() + " unassigned regions."); 2296 } 2297 2298 if (!isRunning()) { 2299 LOG.debug("Stopped! Dropping assign of " + regions.size() + " queued regions."); 2300 return; 2301 } 2302 Threads.sleep(250); 2303 servers = master.getServerManager().createDestinationServersList(); 2304 } 2305 2306 if (!systemHRIs.isEmpty()) { 2307 // System table regions requiring reassignment are present, get region servers 2308 // not available for system table regions 2309 final List<ServerName> excludeServers = getExcludedServersForSystemTable(); 2310 List<ServerName> serversForSysTables = 2311 servers.stream().filter(s -> !excludeServers.contains(s)).collect(Collectors.toList()); 2312 if (serversForSysTables.isEmpty()) { 2313 LOG.warn("Filtering old server versions and the excluded produced an empty set; " 2314 + "instead considering all candidate servers!"); 2315 } 2316 LOG.debug("Processing assignQueue; systemServersCount=" + serversForSysTables.size() 2317 + ", allServersCount=" + servers.size()); 2318 processAssignmentPlans(regions, null, systemHRIs, 2319 serversForSysTables.isEmpty() && !containsBogusAssignments(regions, systemHRIs) 2320 ? servers 2321 : serversForSysTables); 2322 } 2323 2324 processAssignmentPlans(regions, retainMap, userHRIs, servers); 2325 } 2326 2327 private boolean containsBogusAssignments(Map<RegionInfo, RegionStateNode> regions, 2328 List<RegionInfo> hirs) { 2329 for (RegionInfo ri : hirs) { 2330 if ( 2331 regions.get(ri).getRegionLocation() != null 2332 && regions.get(ri).getRegionLocation().equals(LoadBalancer.BOGUS_SERVER_NAME) 2333 ) { 2334 return true; 2335 } 2336 } 2337 return false; 2338 } 2339 2340 private void processAssignmentPlans(final HashMap<RegionInfo, RegionStateNode> regions, 2341 final HashMap<RegionInfo, ServerName> retainMap, final List<RegionInfo> hris, 2342 final List<ServerName> servers) { 2343 boolean isTraceEnabled = LOG.isTraceEnabled(); 2344 if (isTraceEnabled) { 2345 LOG.trace("Available servers count=" + servers.size() + ": " + servers); 2346 } 2347 2348 final LoadBalancer balancer = getBalancer(); 2349 // ask the balancer where to place regions 2350 if (retainMap != null && !retainMap.isEmpty()) { 2351 if (isTraceEnabled) { 2352 LOG.trace("retain assign regions=" + retainMap); 2353 } 2354 try { 2355 acceptPlan(regions, balancer.retainAssignment(retainMap, servers)); 2356 } catch (IOException e) { 2357 LOG.warn("unable to retain assignment", e); 2358 addToPendingAssignment(regions, retainMap.keySet()); 2359 } 2360 } 2361 2362 // TODO: Do we need to split retain and round-robin? 2363 // the retain seems to fallback to round-robin/random if the region is not in the map. 2364 if (!hris.isEmpty()) { 2365 Collections.sort(hris, RegionInfo.COMPARATOR); 2366 if (isTraceEnabled) { 2367 LOG.trace("round robin regions=" + hris); 2368 } 2369 try { 2370 acceptPlan(regions, balancer.roundRobinAssignment(hris, servers)); 2371 } catch (IOException e) { 2372 LOG.warn("unable to round-robin assignment", e); 2373 addToPendingAssignment(regions, hris); 2374 } 2375 } 2376 } 2377 2378 private void acceptPlan(final HashMap<RegionInfo, RegionStateNode> regions, 2379 final Map<ServerName, List<RegionInfo>> plan) throws HBaseIOException { 2380 final ProcedureEvent<?>[] events = new ProcedureEvent[regions.size()]; 2381 final long st = EnvironmentEdgeManager.currentTime(); 2382 2383 if (plan.isEmpty()) { 2384 throw new HBaseIOException("unable to compute plans for regions=" + regions.size()); 2385 } 2386 2387 int evcount = 0; 2388 for (Map.Entry<ServerName, List<RegionInfo>> entry : plan.entrySet()) { 2389 final ServerName server = entry.getKey(); 2390 for (RegionInfo hri : entry.getValue()) { 2391 final RegionStateNode regionNode = regions.get(hri); 2392 regionNode.setRegionLocation(server); 2393 if (server.equals(LoadBalancer.BOGUS_SERVER_NAME) && regionNode.isSystemTable()) { 2394 assignQueueLock.lock(); 2395 try { 2396 pendingAssignQueue.add(regionNode); 2397 } finally { 2398 assignQueueLock.unlock(); 2399 } 2400 } else { 2401 events[evcount++] = regionNode.getProcedureEvent(); 2402 } 2403 } 2404 } 2405 ProcedureEvent.wakeEvents(getProcedureScheduler(), events); 2406 2407 final long et = EnvironmentEdgeManager.currentTime(); 2408 if (LOG.isTraceEnabled()) { 2409 LOG.trace("ASSIGN ACCEPT " + events.length + " -> " + StringUtils.humanTimeDiff(et - st)); 2410 } 2411 } 2412 2413 private void addToPendingAssignment(final HashMap<RegionInfo, RegionStateNode> regions, 2414 final Collection<RegionInfo> pendingRegions) { 2415 assignQueueLock.lock(); 2416 try { 2417 for (RegionInfo hri : pendingRegions) { 2418 pendingAssignQueue.add(regions.get(hri)); 2419 } 2420 } finally { 2421 assignQueueLock.unlock(); 2422 } 2423 } 2424 2425 /** 2426 * For a given cluster with mixed versions of servers, get a list of servers with lower versions, 2427 * where system table regions should not be assigned to. For system table, we must assign regions 2428 * to a server with highest version. However, we can disable this exclusion using config: 2429 * "hbase.min.version.move.system.tables" if checkForMinVersion is true. Detailed explanation 2430 * available with definition of minVersionToMoveSysTables. 2431 * @return List of Excluded servers for System table regions. 2432 */ 2433 public List<ServerName> getExcludedServersForSystemTable() { 2434 // TODO: This should be a cached list kept by the ServerManager rather than calculated on each 2435 // move or system region assign. The RegionServerTracker keeps list of online Servers with 2436 // RegionServerInfo that includes Version. 2437 List<Pair<ServerName, String>> serverList = 2438 master.getServerManager().getOnlineServersList().stream() 2439 .map(s -> new Pair<>(s, master.getRegionServerVersion(s))).collect(Collectors.toList()); 2440 if (serverList.isEmpty()) { 2441 return new ArrayList<>(); 2442 } 2443 String highestVersion = Collections 2444 .max(serverList, (o1, o2) -> VersionInfo.compareVersion(o1.getSecond(), o2.getSecond())) 2445 .getSecond(); 2446 if (!DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG.equals(minVersionToMoveSysTables)) { 2447 int comparedValue = VersionInfo.compareVersion(minVersionToMoveSysTables, highestVersion); 2448 if (comparedValue > 0) { 2449 return new ArrayList<>(); 2450 } 2451 } 2452 return serverList.stream().filter(pair -> !pair.getSecond().equals(highestVersion)) 2453 .map(Pair::getFirst).collect(Collectors.toList()); 2454 } 2455 2456 MasterServices getMaster() { 2457 return master; 2458 } 2459 2460 /** Returns a snapshot of rsReports */ 2461 public Map<ServerName, Set<byte[]>> getRSReports() { 2462 Map<ServerName, Set<byte[]>> rsReportsSnapshot = new HashMap<>(); 2463 synchronized (rsReports) { 2464 rsReports.entrySet().forEach(e -> rsReportsSnapshot.put(e.getKey(), e.getValue())); 2465 } 2466 return rsReportsSnapshot; 2467 } 2468 2469 /** 2470 * Provide regions state count for given table. e.g howmany regions of give table are 2471 * opened/closed/rit etc 2472 * @param tableName TableName 2473 * @return region states count 2474 */ 2475 public RegionStatesCount getRegionStatesCount(TableName tableName) { 2476 int openRegionsCount = 0; 2477 int closedRegionCount = 0; 2478 int ritCount = 0; 2479 int splitRegionCount = 0; 2480 int totalRegionCount = 0; 2481 if (!isTableDisabled(tableName)) { 2482 final List<RegionState> states = regionStates.getTableRegionStates(tableName); 2483 for (RegionState regionState : states) { 2484 if (regionState.isOpened()) { 2485 openRegionsCount++; 2486 } else if (regionState.isClosed()) { 2487 closedRegionCount++; 2488 } else if (regionState.isSplit()) { 2489 splitRegionCount++; 2490 } 2491 } 2492 totalRegionCount = states.size(); 2493 ritCount = totalRegionCount - openRegionsCount - splitRegionCount; 2494 } 2495 return new RegionStatesCount.RegionStatesCountBuilder().setOpenRegions(openRegionsCount) 2496 .setClosedRegions(closedRegionCount).setSplitRegions(splitRegionCount) 2497 .setRegionsInTransition(ritCount).setTotalRegions(totalRegionCount).build(); 2498 } 2499 2500}