001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.procedure2; 019 020import java.io.IOException; 021import java.util.Arrays; 022import java.util.List; 023import java.util.Map; 024import java.util.concurrent.ThreadLocalRandom; 025import org.apache.hadoop.hbase.exceptions.TimeoutIOException; 026import org.apache.hadoop.hbase.metrics.Counter; 027import org.apache.hadoop.hbase.metrics.Histogram; 028import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; 029import org.apache.hadoop.hbase.procedure2.util.StringUtils; 030import org.apache.hadoop.hbase.security.User; 031import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 032import org.apache.hadoop.hbase.util.NonceKey; 033import org.apache.yetus.audience.InterfaceAudience; 034import org.slf4j.Logger; 035import org.slf4j.LoggerFactory; 036 037import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; 038import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; 039 040/** 041 * Base Procedure class responsible for Procedure Metadata; e.g. state, submittedTime, lastUpdate, 042 * stack-indexes, etc. 043 * <p/> 044 * Procedures are run by a {@link ProcedureExecutor} instance. They are submitted and then the 045 * ProcedureExecutor keeps calling {@link #execute(Object)} until the Procedure is done. Execute may 046 * be called multiple times in the case of failure or a restart, so code must be idempotent. The 047 * return from an execute call is either: null to indicate we are done; ourself if there is more to 048 * do; or, a set of sub-procedures that need to be run to completion before the framework resumes 049 * our execution. 050 * <p/> 051 * The ProcedureExecutor keeps its notion of Procedure State in the Procedure itself; e.g. it stamps 052 * the Procedure as INITIALIZING, RUNNABLE, SUCCESS, etc. Here are some of the States defined in the 053 * ProcedureState enum from protos: 054 * <ul> 055 * <li>{@link #isFailed()} A procedure has executed at least once and has failed. The procedure may 056 * or may not have rolled back yet. Any procedure in FAILED state will be eventually moved to 057 * ROLLEDBACK state.</li> 058 * <li>{@link #isSuccess()} A procedure is completed successfully without exception.</li> 059 * <li>{@link #isFinished()} As a procedure in FAILED state will be tried forever for rollback, only 060 * condition when scheduler/ executor will drop procedure from further processing is when procedure 061 * state is ROLLEDBACK or isSuccess() returns true. This is a terminal state of the procedure.</li> 062 * <li>{@link #isWaiting()} - Procedure is in one of the two waiting states 063 * ({@link ProcedureState#WAITING}, {@link ProcedureState#WAITING_TIMEOUT}).</li> 064 * </ul> 065 * NOTE: These states are of the ProcedureExecutor. Procedure implementations in turn can keep their 066 * own state. This can lead to confusion. Try to keep the two distinct. 067 * <p/> 068 * rollback() is called when the procedure or one of the sub-procedures has failed. The rollback 069 * step is supposed to cleanup the resources created during the execute() step. In case of failure 070 * and restart, rollback() may be called multiple times, so again the code must be idempotent. 071 * <p/> 072 * Procedure can be made respect a locking regime. It has acquire/release methods as well as an 073 * {@link #hasLock()}. The lock implementation is up to the implementor. If an entity needs to be 074 * locked for the life of a procedure -- not just the calls to execute -- then implementations 075 * should say so with the {@link #holdLock(Object)} method. 076 * <p/> 077 * And since we need to restore the lock when restarting to keep the logic correct(HBASE-20846), the 078 * implementation is a bit tricky so we add some comments hrre about it. 079 * <ul> 080 * <li>Make {@link #hasLock()} method final, and add a {@link #locked} field in Procedure to record 081 * whether we have the lock. We will set it to {@code true} in 082 * {@link #doAcquireLock(Object, ProcedureStore)} and to {@code false} in 083 * {@link #doReleaseLock(Object, ProcedureStore)}. The sub classes do not need to manage it any 084 * more.</li> 085 * <li>Also added a locked field in the proto message. When storing, the field will be set according 086 * to the return value of {@link #hasLock()}. And when loading, there is a new field in Procedure 087 * called {@link #lockedWhenLoading}. We will set it to {@code true} if the locked field in proto 088 * message is {@code true}.</li> 089 * <li>The reason why we can not set the {@link #locked} field directly to {@code true} by calling 090 * {@link #doAcquireLock(Object, ProcedureStore)} is that, during initialization, most procedures 091 * need to wait until master is initialized. So the solution here is that, we introduced a new 092 * method called {@link #waitInitialized(Object)} in Procedure, and move the wait master initialized 093 * related code from {@link #acquireLock(Object)} to this method. And we added a restoreLock method 094 * to Procedure, if {@link #lockedWhenLoading} is {@code true}, we will call the 095 * {@link #acquireLock(Object)} to get the lock, but do not set {@link #locked} to true. And later 096 * when we call {@link #doAcquireLock(Object, ProcedureStore)} and pass the 097 * {@link #waitInitialized(Object)} check, we will test {@link #lockedWhenLoading}, if it is 098 * {@code true}, when we just set the {@link #locked} field to true and return, without actually 099 * calling the {@link #acquireLock(Object)} method since we have already called it once.</li> 100 * </ul> 101 * <p/> 102 * Procedures can be suspended or put in wait state with a callback that gets executed on 103 * Procedure-specified timeout. See {@link #setTimeout(int)}}, and 104 * {@link #setTimeoutFailure(Object)}. See TestProcedureEvents and the TestTimeoutEventProcedure 105 * class for an example usage. 106 * </p> 107 * <p/> 108 * There are hooks for collecting metrics on submit of the procedure and on finish. See 109 * {@link #updateMetricsOnSubmit(Object)} and {@link #updateMetricsOnFinish(Object, long, boolean)}. 110 */ 111@InterfaceAudience.Private 112public abstract class Procedure<TEnvironment> implements Comparable<Procedure<TEnvironment>> { 113 private static final Logger LOG = LoggerFactory.getLogger(Procedure.class); 114 public static final long NO_PROC_ID = -1; 115 protected static final int NO_TIMEOUT = -1; 116 117 public enum LockState { 118 LOCK_ACQUIRED, // Lock acquired and ready to execute 119 LOCK_YIELD_WAIT, // Lock not acquired, framework needs to yield 120 LOCK_EVENT_WAIT, // Lock not acquired, an event will yield the procedure 121 } 122 123 // Unchanged after initialization 124 private NonceKey nonceKey = null; 125 private String owner = null; 126 private long parentProcId = NO_PROC_ID; 127 private long rootProcId = NO_PROC_ID; 128 private long procId = NO_PROC_ID; 129 private long submittedTime; 130 131 // Runtime state, updated every operation 132 private ProcedureState state = ProcedureState.INITIALIZING; 133 private RemoteProcedureException exception = null; 134 private int[] stackIndexes = null; 135 private int childrenLatch = 0; 136 137 private volatile int timeout = NO_TIMEOUT; 138 private volatile long lastUpdate; 139 140 private volatile byte[] result = null; 141 142 private volatile boolean locked = false; 143 144 private boolean lockedWhenLoading = false; 145 146 /** 147 * Used for override complete of the procedure without actually doing any logic in the procedure. 148 * If bypass is set to true, when executing it will return null when {@link #doExecute(Object)} is 149 * called to finish the procedure and release any locks it may currently hold. The bypass does 150 * cleanup around the Procedure as far as the Procedure framework is concerned. It does not clean 151 * any internal state that the Procedure's themselves may have set. That is for the Procedures to 152 * do themselves when bypass is called. They should override bypass and do their cleanup in the 153 * overridden bypass method (be sure to call the parent bypass to ensure proper processing). 154 * <p> 155 * </p> 156 * Bypassing a procedure is not like aborting. Aborting a procedure will trigger a rollback. And 157 * since the {@link #abort(Object)} method is overrideable Some procedures may have chosen to 158 * ignore the aborting. 159 */ 160 private volatile boolean bypass = false; 161 162 /** 163 * Indicate whether we need to persist the procedure to ProcedureStore after execution. Default to 164 * true, and the implementation can all {@link #skipPersistence()} to let the framework skip the 165 * persistence of the procedure. 166 * <p/> 167 * This is useful when the procedure is in error and you want to retry later. The retry interval 168 * and the number of retries are usually not critical so skip the persistence can save some 169 * resources, and also speed up the restart processing. 170 * <p/> 171 * Notice that this value will be reset to true every time before execution. And when rolling back 172 * we do not test this value. 173 */ 174 private boolean persist = true; 175 176 public boolean isBypass() { 177 return bypass; 178 } 179 180 /** 181 * Set the bypass to true. Only called in 182 * {@link ProcedureExecutor#bypassProcedure(long, long, boolean, boolean)} for now. DO NOT use 183 * this method alone, since we can't just bypass one single procedure. We need to bypass its 184 * ancestor too. If your Procedure has set state, it needs to undo it in here. 185 * @param env Current environment. May be null because of context; e.g. pretty-printing procedure 186 * WALs where there is no 'environment' (and where Procedures that require an 187 * 'environment' won't be run. 188 */ 189 protected void bypass(TEnvironment env) { 190 this.bypass = true; 191 } 192 193 boolean needPersistence() { 194 return persist; 195 } 196 197 void resetPersistence() { 198 persist = true; 199 } 200 201 protected final void skipPersistence() { 202 persist = false; 203 } 204 205 /** 206 * The main code of the procedure. It must be idempotent since execute() may be called multiple 207 * times in case of machine failure in the middle of the execution. 208 * @param env the environment passed to the ProcedureExecutor 209 * @return a set of sub-procedures to run or ourselves if there is more work to do or null if the 210 * procedure is done. 211 * @throws ProcedureYieldException the procedure will be added back to the queue and retried 212 * later. 213 * @throws InterruptedException the procedure will be added back to the queue and retried 214 * later. 215 * @throws ProcedureSuspendedException Signal to the executor that Procedure has suspended itself 216 * and has set itself up waiting for an external event to wake 217 * it back up again. 218 */ 219 protected abstract Procedure<TEnvironment>[] execute(TEnvironment env) 220 throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException; 221 222 /** 223 * The code to undo what was done by the execute() code. It is called when the procedure or one of 224 * the sub-procedures failed or an abort was requested. It should cleanup all the resources 225 * created by the execute() call. The implementation must be idempotent since rollback() may be 226 * called multiple time in case of machine failure in the middle of the execution. 227 * @param env the environment passed to the ProcedureExecutor 228 * @throws IOException temporary failure, the rollback will retry later 229 * @throws InterruptedException the procedure will be added back to the queue and retried later 230 */ 231 protected abstract void rollback(TEnvironment env) throws IOException, InterruptedException; 232 233 /** 234 * The abort() call is asynchronous and each procedure must decide how to deal with it, if they 235 * want to be abortable. The simplest implementation is to have an AtomicBoolean set in the 236 * abort() method and then the execute() will check if the abort flag is set or not. abort() may 237 * be called multiple times from the client, so the implementation must be idempotent. 238 * <p> 239 * NOTE: abort() is not like Thread.interrupt(). It is just a notification that allows the 240 * procedure implementor abort. 241 */ 242 protected abstract boolean abort(TEnvironment env); 243 244 /** 245 * The user-level code of the procedure may have some state to persist (e.g. input arguments or 246 * current position in the processing state) to be able to resume on failure. 247 * @param serializer stores the serializable state 248 */ 249 protected abstract void serializeStateData(ProcedureStateSerializer serializer) 250 throws IOException; 251 252 /** 253 * Called on store load to allow the user to decode the previously serialized state. 254 * @param serializer contains the serialized state 255 */ 256 protected abstract void deserializeStateData(ProcedureStateSerializer serializer) 257 throws IOException; 258 259 /** 260 * The {@link #doAcquireLock(Object, ProcedureStore)} will be split into two steps, first, it will 261 * call us to determine whether we need to wait for initialization, second, it will call 262 * {@link #acquireLock(Object)} to actually handle the lock for this procedure. 263 * <p/> 264 * This is because that when master restarts, we need to restore the lock state for all the 265 * procedures to not break the semantic if {@link #holdLock(Object)} is true. But the 266 * {@link ProcedureExecutor} will be started before the master finish initialization(as it is part 267 * of the initialization!), so we need to split the code into two steps, and when restore, we just 268 * restore the lock part and ignore the waitInitialized part. Otherwise there will be dead lock. 269 * @return true means we need to wait until the environment has been initialized, otherwise true. 270 */ 271 protected boolean waitInitialized(TEnvironment env) { 272 return false; 273 } 274 275 /** 276 * The user should override this method if they need a lock on an Entity. A lock can be anything, 277 * and it is up to the implementor. The Procedure Framework will call this method just before it 278 * invokes {@link #execute(Object)}. It calls {@link #releaseLock(Object)} after the call to 279 * execute. 280 * <p/> 281 * If you need to hold the lock for the life of the Procedure -- i.e. you do not want any other 282 * Procedure interfering while this Procedure is running, see {@link #holdLock(Object)}. 283 * <p/> 284 * Example: in our Master we can execute request in parallel for different tables. We can create 285 * t1 and create t2 and these creates can be executed at the same time. Anything else on t1/t2 is 286 * queued waiting that specific table create to happen. 287 * <p/> 288 * There are 3 LockState: 289 * <ul> 290 * <li>LOCK_ACQUIRED should be returned when the proc has the lock and the proc is ready to 291 * execute.</li> 292 * <li>LOCK_YIELD_WAIT should be returned when the proc has not the lock and the framework should 293 * take care of readding the procedure back to the runnable set for retry</li> 294 * <li>LOCK_EVENT_WAIT should be returned when the proc has not the lock and someone will take 295 * care of readding the procedure back to the runnable set when the lock is available.</li> 296 * </ul> 297 * @return the lock state as described above. 298 */ 299 protected LockState acquireLock(TEnvironment env) { 300 return LockState.LOCK_ACQUIRED; 301 } 302 303 /** 304 * The user should override this method, and release lock if necessary. 305 */ 306 protected void releaseLock(TEnvironment env) { 307 // no-op 308 } 309 310 /** 311 * Used to keep the procedure lock even when the procedure is yielding or suspended. 312 * @return true if the procedure should hold on the lock until completionCleanup() 313 */ 314 protected boolean holdLock(TEnvironment env) { 315 return false; 316 } 317 318 /** 319 * This is used in conjunction with {@link #holdLock(Object)}. If {@link #holdLock(Object)} 320 * returns true, the procedure executor will call acquireLock() once and thereafter not call 321 * {@link #releaseLock(Object)} until the Procedure is done (Normally, it calls release/acquire 322 * around each invocation of {@link #execute(Object)}. 323 * @see #holdLock(Object) 324 * @return true if the procedure has the lock, false otherwise. 325 */ 326 public final boolean hasLock() { 327 return locked; 328 } 329 330 /** 331 * Called when the procedure is loaded for replay. The procedure implementor may use this method 332 * to perform some quick operation before replay. e.g. failing the procedure if the state on 333 * replay may be unknown. 334 */ 335 protected void beforeReplay(TEnvironment env) { 336 // no-op 337 } 338 339 /** 340 * Called when the procedure is ready to be added to the queue after the loading/replay operation. 341 */ 342 protected void afterReplay(TEnvironment env) { 343 // no-op 344 } 345 346 /** 347 * Called when the procedure is marked as completed (success or rollback). The procedure 348 * implementor may use this method to cleanup in-memory states. This operation will not be retried 349 * on failure. If a procedure took a lock, it will have been released when this method runs. 350 */ 351 protected void completionCleanup(TEnvironment env) { 352 // no-op 353 } 354 355 /** 356 * By default, the procedure framework/executor will try to run procedures start to finish. Return 357 * true to make the executor yield between each execution step to give other procedures a chance 358 * to run. 359 * @param env the environment passed to the ProcedureExecutor 360 * @return Return true if the executor should yield on completion of an execution step. Defaults 361 * to return false. 362 */ 363 protected boolean isYieldAfterExecutionStep(TEnvironment env) { 364 return false; 365 } 366 367 /** 368 * By default, the executor will keep the procedure result around util the eviction TTL is 369 * expired. The client can cut down the waiting time by requesting that the result is removed from 370 * the executor. In case of system started procedure, we can force the executor to auto-ack. 371 * @param env the environment passed to the ProcedureExecutor 372 * @return true if the executor should wait the client ack for the result. Defaults to return 373 * true. 374 */ 375 protected boolean shouldWaitClientAck(TEnvironment env) { 376 return true; 377 } 378 379 /** 380 * Override this method to provide procedure specific counters for submitted count, failed count 381 * and time histogram. 382 * @param env The environment passed to the procedure executor 383 * @return Container object for procedure related metric 384 */ 385 protected ProcedureMetrics getProcedureMetrics(TEnvironment env) { 386 return null; 387 } 388 389 /** 390 * This function will be called just when procedure is submitted for execution. Override this 391 * method to update the metrics at the beginning of the procedure. The default implementation 392 * updates submitted counter if {@link #getProcedureMetrics(Object)} returns non-null 393 * {@link ProcedureMetrics}. 394 */ 395 protected void updateMetricsOnSubmit(TEnvironment env) { 396 ProcedureMetrics metrics = getProcedureMetrics(env); 397 if (metrics == null) { 398 return; 399 } 400 401 Counter submittedCounter = metrics.getSubmittedCounter(); 402 if (submittedCounter != null) { 403 submittedCounter.increment(); 404 } 405 } 406 407 /** 408 * This function will be called just after procedure execution is finished. Override this method 409 * to update metrics at the end of the procedure. If {@link #getProcedureMetrics(Object)} returns 410 * non-null {@link ProcedureMetrics}, the default implementation adds runtime of a procedure to a 411 * time histogram for successfully completed procedures. Increments failed counter for failed 412 * procedures. 413 * <p/> 414 * TODO: As any of the sub-procedures on failure rolls back all procedures in the stack, including 415 * successfully finished siblings, this function may get called twice in certain cases for certain 416 * procedures. Explore further if this can be called once. 417 * @param env The environment passed to the procedure executor 418 * @param runtime Runtime of the procedure in milliseconds 419 * @param success true if procedure is completed successfully 420 */ 421 protected void updateMetricsOnFinish(TEnvironment env, long runtime, boolean success) { 422 ProcedureMetrics metrics = getProcedureMetrics(env); 423 if (metrics == null) { 424 return; 425 } 426 427 if (success) { 428 Histogram timeHisto = metrics.getTimeHisto(); 429 if (timeHisto != null) { 430 timeHisto.update(runtime); 431 } 432 } else { 433 Counter failedCounter = metrics.getFailedCounter(); 434 if (failedCounter != null) { 435 failedCounter.increment(); 436 } 437 } 438 } 439 440 @Override 441 public String toString() { 442 // Return the simple String presentation of the procedure. 443 return toStringSimpleSB().toString(); 444 } 445 446 /** 447 * Build the StringBuilder for the simple form of procedure string. 448 * @return the StringBuilder 449 */ 450 protected StringBuilder toStringSimpleSB() { 451 final StringBuilder sb = new StringBuilder(); 452 453 sb.append("pid="); 454 sb.append(getProcId()); 455 456 if (hasParent()) { 457 sb.append(", ppid="); 458 sb.append(getParentProcId()); 459 } 460 461 /* 462 * TODO Enable later when this is being used. Currently owner not used. if (hasOwner()) { 463 * sb.append(", owner="); sb.append(getOwner()); } 464 */ 465 466 sb.append(", state="); // pState for Procedure State as opposed to any other kind. 467 toStringState(sb); 468 469 sb.append(", hasLock=").append(locked); 470 471 if (bypass) { 472 sb.append(", bypass=").append(bypass); 473 } 474 475 if (hasException()) { 476 sb.append(", exception=" + getException()); 477 } 478 479 sb.append("; "); 480 toStringClassDetails(sb); 481 482 return sb; 483 } 484 485 /** 486 * Extend the toString() information with more procedure details 487 */ 488 public String toStringDetails() { 489 final StringBuilder sb = toStringSimpleSB(); 490 491 sb.append(" submittedTime="); 492 sb.append(getSubmittedTime()); 493 494 sb.append(", lastUpdate="); 495 sb.append(getLastUpdate()); 496 497 final int[] stackIndices = getStackIndexes(); 498 if (stackIndices != null) { 499 sb.append("\n"); 500 sb.append("stackIndexes="); 501 sb.append(Arrays.toString(stackIndices)); 502 } 503 504 return sb.toString(); 505 } 506 507 protected String toStringClass() { 508 StringBuilder sb = new StringBuilder(); 509 toStringClassDetails(sb); 510 return sb.toString(); 511 } 512 513 /** 514 * Called from {@link #toString()} when interpolating {@link Procedure} State. Allows decorating 515 * generic Procedure State with Procedure particulars. 516 * @param builder Append current {@link ProcedureState} 517 */ 518 protected void toStringState(StringBuilder builder) { 519 builder.append(getState()); 520 } 521 522 /** 523 * Extend the toString() information with the procedure details e.g. className and parameters 524 * @param builder the string builder to use to append the proc specific information 525 */ 526 protected void toStringClassDetails(StringBuilder builder) { 527 builder.append(getClass().getName()); 528 } 529 530 // ========================================================================== 531 // Those fields are unchanged after initialization. 532 // 533 // Each procedure will get created from the user or during 534 // ProcedureExecutor.start() during the load() phase and then submitted 535 // to the executor. these fields will never be changed after initialization 536 // ========================================================================== 537 public long getProcId() { 538 return procId; 539 } 540 541 public boolean hasParent() { 542 return parentProcId != NO_PROC_ID; 543 } 544 545 public long getParentProcId() { 546 return parentProcId; 547 } 548 549 public long getRootProcId() { 550 return rootProcId; 551 } 552 553 public String getProcName() { 554 return toStringClass(); 555 } 556 557 public NonceKey getNonceKey() { 558 return nonceKey; 559 } 560 561 public long getSubmittedTime() { 562 return submittedTime; 563 } 564 565 public String getOwner() { 566 return owner; 567 } 568 569 public boolean hasOwner() { 570 return owner != null; 571 } 572 573 /** 574 * Called by the ProcedureExecutor to assign the ID to the newly created procedure. 575 */ 576 protected void setProcId(long procId) { 577 this.procId = procId; 578 this.submittedTime = EnvironmentEdgeManager.currentTime(); 579 setState(ProcedureState.RUNNABLE); 580 } 581 582 /** 583 * Called by the ProcedureExecutor to assign the parent to the newly created procedure. 584 */ 585 protected void setParentProcId(long parentProcId) { 586 this.parentProcId = parentProcId; 587 } 588 589 protected void setRootProcId(long rootProcId) { 590 this.rootProcId = rootProcId; 591 } 592 593 /** 594 * Called by the ProcedureExecutor to set the value to the newly created procedure. 595 */ 596 protected void setNonceKey(NonceKey nonceKey) { 597 this.nonceKey = nonceKey; 598 } 599 600 public void setOwner(String owner) { 601 this.owner = StringUtils.isEmpty(owner) ? null : owner; 602 } 603 604 public void setOwner(User owner) { 605 assert owner != null : "expected owner to be not null"; 606 setOwner(owner.getShortName()); 607 } 608 609 /** 610 * Called on store load to initialize the Procedure internals after the creation/deserialization. 611 */ 612 protected void setSubmittedTime(long submittedTime) { 613 this.submittedTime = submittedTime; 614 } 615 616 // ========================================================================== 617 // runtime state - timeout related 618 // ========================================================================== 619 /** 620 * @param timeout timeout interval in msec 621 */ 622 protected void setTimeout(int timeout) { 623 this.timeout = timeout; 624 } 625 626 public boolean hasTimeout() { 627 return timeout != NO_TIMEOUT; 628 } 629 630 /** Returns the timeout in msec */ 631 public int getTimeout() { 632 return timeout; 633 } 634 635 /** 636 * Called on store load to initialize the Procedure internals after the creation/deserialization. 637 */ 638 protected void setLastUpdate(long lastUpdate) { 639 this.lastUpdate = lastUpdate; 640 } 641 642 /** 643 * Called by ProcedureExecutor after each time a procedure step is executed. 644 */ 645 protected void updateTimestamp() { 646 this.lastUpdate = EnvironmentEdgeManager.currentTime(); 647 } 648 649 public long getLastUpdate() { 650 return lastUpdate; 651 } 652 653 /** 654 * Timeout of the next timeout. Called by the ProcedureExecutor if the procedure has timeout set 655 * and the procedure is in the waiting queue. 656 * @return the timestamp of the next timeout. 657 */ 658 protected long getTimeoutTimestamp() { 659 return getLastUpdate() + getTimeout(); 660 } 661 662 // ========================================================================== 663 // runtime state 664 // ========================================================================== 665 /** Returns the time elapsed between the last update and the start time of the procedure. */ 666 public long elapsedTime() { 667 return getLastUpdate() - getSubmittedTime(); 668 } 669 670 /** Returns the serialized result if any, otherwise null */ 671 public byte[] getResult() { 672 return result; 673 } 674 675 /** 676 * The procedure may leave a "result" on completion. 677 * @param result the serialized result that will be passed to the client 678 */ 679 protected void setResult(byte[] result) { 680 this.result = result; 681 } 682 683 /** 684 * Will only be called when loading procedures from procedure store, where we need to record 685 * whether the procedure has already held a lock. Later we will call {@link #restoreLock(Object)} 686 * to actually acquire the lock. 687 */ 688 final void lockedWhenLoading() { 689 this.lockedWhenLoading = true; 690 } 691 692 /** 693 * Can only be called when restarting, before the procedure actually being executed, as after we 694 * actually call the {@link #doAcquireLock(Object, ProcedureStore)} method, we will reset 695 * {@link #lockedWhenLoading} to false. 696 * <p/> 697 * Now it is only used in the ProcedureScheduler to determine whether we should put a Procedure in 698 * front of a queue. 699 */ 700 public boolean isLockedWhenLoading() { 701 return lockedWhenLoading; 702 } 703 704 // ============================================================================================== 705 // Runtime state, updated every operation by the ProcedureExecutor 706 // 707 // There is always 1 thread at the time operating on the state of the procedure. 708 // The ProcedureExecutor may check and set states, or some Procecedure may 709 // update its own state. but no concurrent updates. we use synchronized here 710 // just because the procedure can get scheduled on different executor threads on each step. 711 // ============================================================================================== 712 713 /** Returns true if the procedure is in a RUNNABLE state. */ 714 public synchronized boolean isRunnable() { 715 return state == ProcedureState.RUNNABLE; 716 } 717 718 public synchronized boolean isInitializing() { 719 return state == ProcedureState.INITIALIZING; 720 } 721 722 /** Returns true if the procedure has failed. It may or may not have rolled back. */ 723 public synchronized boolean isFailed() { 724 return state == ProcedureState.FAILED || state == ProcedureState.ROLLEDBACK; 725 } 726 727 /** Returns true if the procedure is finished successfully. */ 728 public synchronized boolean isSuccess() { 729 return state == ProcedureState.SUCCESS && !hasException(); 730 } 731 732 /** 733 * @return true if the procedure is finished. The Procedure may be completed successfully or 734 * rolledback. 735 */ 736 public synchronized boolean isFinished() { 737 return isSuccess() || state == ProcedureState.ROLLEDBACK; 738 } 739 740 /** Returns true if the procedure is waiting for a child to finish or for an external event. */ 741 public synchronized boolean isWaiting() { 742 switch (state) { 743 case WAITING: 744 case WAITING_TIMEOUT: 745 return true; 746 default: 747 break; 748 } 749 return false; 750 } 751 752 protected synchronized void setState(final ProcedureState state) { 753 this.state = state; 754 updateTimestamp(); 755 } 756 757 public synchronized ProcedureState getState() { 758 return state; 759 } 760 761 protected void setFailure(final String source, final Throwable cause) { 762 setFailure(new RemoteProcedureException(source, cause)); 763 } 764 765 protected synchronized void setFailure(final RemoteProcedureException exception) { 766 this.exception = exception; 767 if (!isFinished()) { 768 setState(ProcedureState.FAILED); 769 } 770 } 771 772 protected void setAbortFailure(final String source, final String msg) { 773 setFailure(source, new ProcedureAbortedException(msg)); 774 } 775 776 /** 777 * Called by the ProcedureExecutor when the timeout set by setTimeout() is expired. 778 * <p/> 779 * Another usage for this method is to implement retrying. A procedure can set the state to 780 * {@code WAITING_TIMEOUT} by calling {@code setState} method, and throw a 781 * {@link ProcedureSuspendedException} to halt the execution of the procedure, and do not forget a 782 * call {@link #setTimeout(int)} method to set the timeout. And you should also override this 783 * method to wake up the procedure, and also return false to tell the ProcedureExecutor that the 784 * timeout event has been handled. 785 * @return true to let the framework handle the timeout as abort, false in case the procedure 786 * handled the timeout itself. 787 */ 788 protected synchronized boolean setTimeoutFailure(TEnvironment env) { 789 if (state == ProcedureState.WAITING_TIMEOUT) { 790 long timeDiff = EnvironmentEdgeManager.currentTime() - lastUpdate; 791 setFailure("ProcedureExecutor", 792 new TimeoutIOException("Operation timed out after " + StringUtils.humanTimeDiff(timeDiff))); 793 return true; 794 } 795 return false; 796 } 797 798 public synchronized boolean hasException() { 799 return exception != null; 800 } 801 802 public synchronized RemoteProcedureException getException() { 803 return exception; 804 } 805 806 /** 807 * Called by the ProcedureExecutor on procedure-load to restore the latch state 808 */ 809 protected synchronized void setChildrenLatch(int numChildren) { 810 this.childrenLatch = numChildren; 811 if (LOG.isTraceEnabled()) { 812 LOG.trace("CHILD LATCH INCREMENT SET " + this.childrenLatch, new Throwable(this.toString())); 813 } 814 } 815 816 /** 817 * Called by the ProcedureExecutor on procedure-load to restore the latch state 818 */ 819 protected synchronized void incChildrenLatch() { 820 // TODO: can this be inferred from the stack? I think so... 821 this.childrenLatch++; 822 if (LOG.isTraceEnabled()) { 823 LOG.trace("CHILD LATCH INCREMENT " + this.childrenLatch, new Throwable(this.toString())); 824 } 825 } 826 827 /** 828 * Called by the ProcedureExecutor to notify that one of the sub-procedures has completed. 829 */ 830 private synchronized boolean childrenCountDown() { 831 assert childrenLatch > 0 : this; 832 boolean b = --childrenLatch == 0; 833 if (LOG.isTraceEnabled()) { 834 LOG.trace("CHILD LATCH DECREMENT " + childrenLatch, new Throwable(this.toString())); 835 } 836 return b; 837 } 838 839 /** 840 * Try to set this procedure into RUNNABLE state. Succeeds if all subprocedures/children are done. 841 * @return True if we were able to move procedure to RUNNABLE state. 842 */ 843 synchronized boolean tryRunnable() { 844 // Don't use isWaiting in the below; it returns true for WAITING and WAITING_TIMEOUT 845 if (getState() == ProcedureState.WAITING && childrenCountDown()) { 846 setState(ProcedureState.RUNNABLE); 847 return true; 848 } else { 849 return false; 850 } 851 } 852 853 protected synchronized boolean hasChildren() { 854 return childrenLatch > 0; 855 } 856 857 protected synchronized int getChildrenLatch() { 858 return childrenLatch; 859 } 860 861 /** 862 * Called by the RootProcedureState on procedure execution. Each procedure store its stack-index 863 * positions. 864 */ 865 protected synchronized void addStackIndex(final int index) { 866 if (stackIndexes == null) { 867 stackIndexes = new int[] { index }; 868 } else { 869 int count = stackIndexes.length; 870 stackIndexes = Arrays.copyOf(stackIndexes, count + 1); 871 stackIndexes[count] = index; 872 } 873 } 874 875 protected synchronized boolean removeStackIndex() { 876 if (stackIndexes != null && stackIndexes.length > 1) { 877 stackIndexes = Arrays.copyOf(stackIndexes, stackIndexes.length - 1); 878 return false; 879 } else { 880 stackIndexes = null; 881 return true; 882 } 883 } 884 885 /** 886 * Called on store load to initialize the Procedure internals after the creation/deserialization. 887 */ 888 protected synchronized void setStackIndexes(final List<Integer> stackIndexes) { 889 this.stackIndexes = new int[stackIndexes.size()]; 890 for (int i = 0; i < this.stackIndexes.length; ++i) { 891 this.stackIndexes[i] = stackIndexes.get(i); 892 } 893 } 894 895 protected synchronized boolean wasExecuted() { 896 return stackIndexes != null; 897 } 898 899 protected synchronized int[] getStackIndexes() { 900 return stackIndexes; 901 } 902 903 // ========================================================================== 904 // Internal methods - called by the ProcedureExecutor 905 // ========================================================================== 906 907 /** 908 * Internal method called by the ProcedureExecutor that starts the user-level code execute(). 909 * @throws ProcedureSuspendedException This is used when procedure wants to halt processing and 910 * skip out without changing states or releasing any locks 911 * held. 912 */ 913 protected Procedure<TEnvironment>[] doExecute(TEnvironment env) 914 throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException { 915 try { 916 updateTimestamp(); 917 if (bypass) { 918 LOG.info("{} bypassed, returning null to finish it", this); 919 return null; 920 } 921 return execute(env); 922 } finally { 923 updateTimestamp(); 924 } 925 } 926 927 /** 928 * Internal method called by the ProcedureExecutor that starts the user-level code rollback(). 929 */ 930 protected void doRollback(TEnvironment env) throws IOException, InterruptedException { 931 try { 932 updateTimestamp(); 933 if (bypass) { 934 LOG.info("{} bypassed, skipping rollback", this); 935 return; 936 } 937 rollback(env); 938 } finally { 939 updateTimestamp(); 940 } 941 } 942 943 final void restoreLock(TEnvironment env) { 944 if (!lockedWhenLoading) { 945 LOG.debug("{} didn't hold the lock before restarting, skip acquiring lock.", this); 946 return; 947 } 948 949 if (isFinished()) { 950 LOG.debug("{} is already finished, skip acquiring lock.", this); 951 return; 952 } 953 954 if (isBypass()) { 955 LOG.debug("{} is already bypassed, skip acquiring lock.", this); 956 return; 957 } 958 // this can happen if the parent stores the sub procedures but before it can 959 // release its lock, the master restarts 960 if (getState() == ProcedureState.WAITING && !holdLock(env)) { 961 LOG.debug("{} is in WAITING STATE, and holdLock=false, skip acquiring lock.", this); 962 lockedWhenLoading = false; 963 return; 964 } 965 LOG.debug("{} held the lock before restarting, call acquireLock to restore it.", this); 966 LockState state = acquireLock(env); 967 assert state == LockState.LOCK_ACQUIRED; 968 } 969 970 /** 971 * Internal method called by the ProcedureExecutor that starts the user-level code acquireLock(). 972 */ 973 final LockState doAcquireLock(TEnvironment env, ProcedureStore store) { 974 if (waitInitialized(env)) { 975 return LockState.LOCK_EVENT_WAIT; 976 } 977 if (lockedWhenLoading) { 978 // reset it so we will not consider it anymore 979 lockedWhenLoading = false; 980 locked = true; 981 // Here we return without persist the locked state, as lockedWhenLoading is true means 982 // that the locked field of the procedure stored in procedure store is true, so we do not need 983 // to store it again. 984 return LockState.LOCK_ACQUIRED; 985 } 986 LockState state = acquireLock(env); 987 if (state == LockState.LOCK_ACQUIRED) { 988 locked = true; 989 // persist that we have held the lock. This must be done before we actually execute the 990 // procedure, otherwise when restarting, we may consider the procedure does not have a lock, 991 // but it may have already done some changes as we have already executed it, and if another 992 // procedure gets the lock, then the semantic will be broken if the holdLock is true, as we do 993 // not expect that another procedure can be executed in the middle. 994 store.update(this); 995 } 996 return state; 997 } 998 999 /** 1000 * Internal method called by the ProcedureExecutor that starts the user-level code releaseLock(). 1001 */ 1002 final void doReleaseLock(TEnvironment env, ProcedureStore store) { 1003 locked = false; 1004 // persist that we have released the lock. This must be done before we actually release the 1005 // lock. Another procedure may take this lock immediately after we release the lock, and if we 1006 // crash before persist the information that we have already released the lock, then when 1007 // restarting there will be two procedures which both have the lock and cause problems. 1008 if (getState() != ProcedureState.ROLLEDBACK) { 1009 // If the state is ROLLEDBACK, it means that we have already deleted the procedure from 1010 // procedure store, so do not need to log the release operation any more. 1011 store.update(this); 1012 } 1013 releaseLock(env); 1014 } 1015 1016 protected final ProcedureSuspendedException suspend(int timeoutMillis, boolean jitter) 1017 throws ProcedureSuspendedException { 1018 if (jitter) { 1019 // 10% possible jitter 1020 double add = (double) timeoutMillis * ThreadLocalRandom.current().nextDouble(0.1); 1021 timeoutMillis += add; 1022 } 1023 setTimeout(timeoutMillis); 1024 setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT); 1025 skipPersistence(); 1026 throw new ProcedureSuspendedException(); 1027 } 1028 1029 @Override 1030 public int compareTo(final Procedure<TEnvironment> other) { 1031 return Long.compare(getProcId(), other.getProcId()); 1032 } 1033 1034 // ========================================================================== 1035 // misc utils 1036 // ========================================================================== 1037 1038 /** 1039 * Get an hashcode for the specified Procedure ID 1040 * @return the hashcode for the specified procId 1041 */ 1042 public static long getProcIdHashCode(long procId) { 1043 long h = procId; 1044 h ^= h >> 16; 1045 h *= 0x85ebca6b; 1046 h ^= h >> 13; 1047 h *= 0xc2b2ae35; 1048 h ^= h >> 16; 1049 return h; 1050 } 1051 1052 /** 1053 * Helper to lookup the root Procedure ID given a specified procedure. 1054 */ 1055 protected static <T> Long getRootProcedureId(Map<Long, Procedure<T>> procedures, 1056 Procedure<T> proc) { 1057 while (proc.hasParent()) { 1058 proc = procedures.get(proc.getParentProcId()); 1059 if (proc == null) { 1060 return null; 1061 } 1062 } 1063 return proc.getProcId(); 1064 } 1065 1066 /** 1067 * @param a the first procedure to be compared. 1068 * @param b the second procedure to be compared. 1069 * @return true if the two procedures have the same parent 1070 */ 1071 public static boolean haveSameParent(Procedure<?> a, Procedure<?> b) { 1072 return a.hasParent() && b.hasParent() && (a.getParentProcId() == b.getParentProcId()); 1073 } 1074}