001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.procedure2; 019 020import java.io.IOException; 021import java.util.concurrent.CompletableFuture; 022import java.util.concurrent.ExecutorService; 023import java.util.concurrent.atomic.AtomicInteger; 024import java.util.function.Consumer; 025import java.util.function.Supplier; 026import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 027import org.apache.hadoop.hbase.util.FutureUtils; 028import org.apache.hadoop.hbase.util.IdLock; 029import org.apache.yetus.audience.InterfaceAudience; 030import org.slf4j.Logger; 031import org.slf4j.LoggerFactory; 032 033/** 034 * A helper class for switching procedure out(yielding) while it is doing some time consuming 035 * operation, such as updating meta, where we can get a {@link CompletableFuture} about the 036 * operation. 037 */ 038@InterfaceAudience.Private 039public final class ProcedureFutureUtil { 040 041 private static final Logger LOG = LoggerFactory.getLogger(ProcedureFutureUtil.class); 042 043 private ProcedureFutureUtil() { 044 } 045 046 public static boolean checkFuture(Procedure<?> proc, Supplier<CompletableFuture<Void>> getFuture, 047 Consumer<CompletableFuture<Void>> setFuture, Runnable actionAfterDone) throws IOException { 048 CompletableFuture<Void> future = getFuture.get(); 049 if (future == null) { 050 return false; 051 } 052 // reset future 053 setFuture.accept(null); 054 FutureUtils.get(future); 055 actionAfterDone.run(); 056 return true; 057 } 058 059 public static void suspendIfNecessary(Procedure<?> proc, 060 Consumer<CompletableFuture<Void>> setFuture, CompletableFuture<Void> future, 061 MasterProcedureEnv env, Runnable actionAfterDone) 062 throws IOException, ProcedureSuspendedException { 063 AtomicInteger completed = new AtomicInteger(0); 064 // This is for testing. In ProcedureTestingUtility, we will restart a ProcedureExecutor and 065 // reuse it, for performance, so we need to make sure that all the procedure have been stopped. 066 // But here, the callback of this future is not executed in a PEWorker, so in ProcedureExecutor 067 // we have no way to stop it. So here, we will get the asyncTaskExecutor first, in the PEWorker 068 // thread, where the ProcedureExecutor should have not been stopped yet, then when calling the 069 // callback, if the ProcedureExecutor have already been stopped and restarted, the 070 // asyncTaskExecutor will also be shutdown so we can not add anything back to the scheduler. 071 ExecutorService asyncTaskExecutor = env.getAsyncTaskExecutor(); 072 FutureUtils.addListener(future, (r, e) -> { 073 if (completed.compareAndSet(0, 1)) { 074 // this means the future has already been completed, as we have not execute the below 075 // compareAndSet(0, 2) yet, so here we just set completed to true without doing anything 076 LOG.debug("The future has completed while adding callback, give up suspending procedure {}", 077 proc); 078 return; 079 } 080 LOG.debug("Going to wake up procedure {} because future has completed", proc); 081 // This callback may be called inside netty's event loop, so we should not block it for a long 082 // time. The worker executor will hold the execution lock while executing the procedure, and 083 // we may persist the procedure state inside the lock, which is a time consuming operation. 084 // And what makes things worse is that, we persist procedure state to master local region, 085 // where the AsyncFSWAL implementation will use the same netty's event loop for dealing with 086 // I/O, which could even cause dead lock. 087 asyncTaskExecutor.execute(() -> wakeUp(proc, env)); 088 }); 089 if (completed.compareAndSet(0, 2)) { 090 // suspend the procedure 091 setFuture.accept(future); 092 suspend(proc); 093 } else { 094 // if completed has been set to 1, i.e, not 0, it means the future has already been completed, 095 // so we give up suspending and call the done action directly. 096 FutureUtils.get(future); 097 actionAfterDone.run(); 098 } 099 } 100 101 public static void suspend(Procedure<?> proc) throws ProcedureSuspendedException { 102 proc.skipPersistence(); 103 throw new ProcedureSuspendedException(); 104 } 105 106 public static void wakeUp(Procedure<?> proc, MasterProcedureEnv env) { 107 // should acquire procedure execution lock to make sure that the procedure executor has 108 // finished putting this procedure to the WAITING_TIMEOUT state, otherwise there could be 109 // race and cause unexpected result 110 IdLock procLock = env.getMasterServices().getMasterProcedureExecutor().getProcExecutionLock(); 111 IdLock.Entry lockEntry; 112 try { 113 lockEntry = procLock.getLockEntry(proc.getProcId()); 114 } catch (IOException e) { 115 LOG.error("Error while acquiring execution lock for procedure {}" 116 + " when trying to wake it up, aborting...", proc, e); 117 env.getMasterServices().abort("Can not acquire procedure execution lock", e); 118 return; 119 } 120 try { 121 env.getProcedureScheduler().addFront(proc); 122 } finally { 123 procLock.releaseLockEntry(lockEntry); 124 } 125 } 126}