001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase.util;
020
021import java.io.InterruptedIOException;
022import java.io.IOException;
023import java.lang.reflect.Constructor;
024import java.lang.reflect.InvocationTargetException;
025import java.util.List;
026import java.util.concurrent.TimeUnit;
027import java.util.function.Supplier;
028
029import org.apache.yetus.audience.InterfaceAudience;
030import org.slf4j.Logger;
031import org.slf4j.LoggerFactory;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.hbase.master.HMaster;
034import org.apache.hadoop.hbase.regionserver.HRegionServer;
035
036/**
037 * Utility used running a cluster all in the one JVM.
038 */
039@InterfaceAudience.Private
040public class JVMClusterUtil {
041  private static final Logger LOG = LoggerFactory.getLogger(JVMClusterUtil.class);
042
043  /**
044   * Datastructure to hold RegionServer Thread and RegionServer instance
045   */
046  public static class RegionServerThread extends Thread {
047    private final HRegionServer regionServer;
048
049    public RegionServerThread(final HRegionServer r, final int index) {
050      super(r, "RS:" + index + ";" + r.getServerName().toShortString());
051      this.regionServer = r;
052    }
053
054    /** @return the region server */
055    public HRegionServer getRegionServer() {
056      return this.regionServer;
057    }
058
059    /**
060     * Block until the region server has come online, indicating it is ready
061     * to be used.
062     */
063    public void waitForServerOnline() {
064      // The server is marked online after the init method completes inside of
065      // the HRS#run method.  HRS#init can fail for whatever region.  In those
066      // cases, we'll jump out of the run without setting online flag.  Check
067      // stopRequested so we don't wait here a flag that will never be flipped.
068      regionServer.waitForServerOnline();
069    }
070  }
071
072  /**
073   * Creates a {@link RegionServerThread}.
074   * Call 'start' on the returned thread to make it run.
075   * @param c Configuration to use.
076   * @param hrsc Class to create.
077   * @param index Used distinguishing the object returned.
078   * @throws IOException
079   * @return Region server added.
080   */
081  public static JVMClusterUtil.RegionServerThread createRegionServerThread(final Configuration c,
082      final Class<? extends HRegionServer> hrsc, final int index) throws IOException {
083    HRegionServer server;
084    try {
085      Constructor<? extends HRegionServer> ctor = hrsc.getConstructor(Configuration.class);
086      ctor.setAccessible(true);
087      server = ctor.newInstance(c);
088    } catch (InvocationTargetException ite) {
089      Throwable target = ite.getTargetException();
090      throw new RuntimeException("Failed construction of RegionServer: " +
091        hrsc.toString() + ((target.getCause() != null)?
092          target.getCause().getMessage(): ""), target);
093    } catch (Exception e) {
094      IOException ioe = new IOException();
095      ioe.initCause(e);
096      throw ioe;
097    }
098    return new JVMClusterUtil.RegionServerThread(server, index);
099  }
100
101
102  /**
103   * Datastructure to hold Master Thread and Master instance
104   */
105  public static class MasterThread extends Thread {
106    private final HMaster master;
107
108    public MasterThread(final HMaster m, final int index) {
109      super(m, "M:" + index + ";" + m.getServerName().toShortString());
110      this.master = m;
111    }
112
113    /** @return the master */
114    public HMaster getMaster() {
115      return this.master;
116    }
117  }
118
119  /**
120   * Creates a {@link MasterThread}.
121   * Call 'start' on the returned thread to make it run.
122   * @param c Configuration to use.
123   * @param hmc Class to create.
124   * @param index Used distinguishing the object returned.
125   * @throws IOException
126   * @return Master added.
127   */
128  public static JVMClusterUtil.MasterThread createMasterThread(final Configuration c,
129      final Class<? extends HMaster> hmc, final int index) throws IOException {
130    HMaster server;
131    try {
132      server = hmc.getConstructor(Configuration.class).newInstance(c);
133    } catch (InvocationTargetException ite) {
134      Throwable target = ite.getTargetException();
135      throw new RuntimeException("Failed construction of Master: " +
136        hmc.toString() + ((target.getCause() != null)?
137          target.getCause().getMessage(): ""), target);
138    } catch (Exception e) {
139      IOException ioe = new IOException();
140      ioe.initCause(e);
141      throw ioe;
142    }
143    return new JVMClusterUtil.MasterThread(server, index);
144  }
145
146  private static JVMClusterUtil.MasterThread findActiveMaster(
147    List<JVMClusterUtil.MasterThread> masters) {
148    for (JVMClusterUtil.MasterThread t : masters) {
149      if (t.master.isActiveMaster()) {
150        return t;
151      }
152    }
153
154    return null;
155  }
156
157  /**
158   * Start the cluster.  Waits until there is a primary master initialized
159   * and returns its address.
160   * @param masters
161   * @param regionservers
162   * @return Address to use contacting primary master.
163   */
164  public static String startup(final List<JVMClusterUtil.MasterThread> masters,
165      final List<JVMClusterUtil.RegionServerThread> regionservers) throws IOException {
166    // Implementation note: This method relies on timed sleeps in a loop. It's not great, and
167    // should probably be re-written to use actual synchronization objects, but it's ok for now
168
169    Configuration configuration = null;
170
171    if (masters == null || masters.isEmpty()) {
172      return null;
173    }
174
175    for (JVMClusterUtil.MasterThread t : masters) {
176      configuration = t.getMaster().getConfiguration();
177      t.start();
178    }
179
180    // Wait for an active master
181    //  having an active master before starting the region threads allows
182    //  then to succeed on their connection to master
183    final int startTimeout = configuration != null ? Integer.parseInt(
184        configuration.get("hbase.master.start.timeout.localHBaseCluster", "30000")) : 30000;
185    waitForEvent(startTimeout, "active", () -> findActiveMaster(masters) != null);
186
187    if (regionservers != null) {
188      for (JVMClusterUtil.RegionServerThread t: regionservers) {
189        t.start();
190      }
191    }
192
193    // Wait for an active master to be initialized (implies being master)
194    //  with this, when we return the cluster is complete
195    final int initTimeout = configuration != null ? Integer.parseInt(
196        configuration.get("hbase.master.init.timeout.localHBaseCluster", "200000")) : 200000;
197    waitForEvent(initTimeout, "initialized", () -> {
198        JVMClusterUtil.MasterThread t = findActiveMaster(masters);
199        // master thread should never be null at this point, but let's keep the check anyway
200        return t != null && t.master.isInitialized();
201      }
202    );
203
204    return findActiveMaster(masters).master.getServerName().toString();
205  }
206
207  /**
208   * Utility method to wait some time for an event to occur, and then return control to the caller.
209   * @param millis How long to wait, in milliseconds.
210   * @param action The action that we are waiting for. Will be used in log message if the event
211   *               does not occur.
212   * @param check A Supplier that will be checked periodically to produce an updated true/false
213   *              result indicating if the expected event has happened or not.
214   * @throws InterruptedIOException If we are interrupted while waiting for the event.
215   * @throws RuntimeException If we reach the specified timeout while waiting for the event.
216   */
217  private static void waitForEvent(long millis, String action, Supplier<Boolean> check)
218      throws InterruptedIOException {
219    long end = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(millis);
220
221    while (true) {
222      if (check.get()) {
223        return;
224      }
225
226      if (System.nanoTime() > end) {
227        String msg = "Master not " + action + " after " + millis + "ms";
228        Threads.printThreadInfo(System.out, "Thread dump because: " + msg);
229        throw new RuntimeException(msg);
230      }
231
232      try {
233        Thread.sleep(100);
234      } catch (InterruptedException e) {
235        throw (InterruptedIOException)new InterruptedIOException().initCause(e);
236      }
237    }
238
239  }
240
241  /**
242   * @param masters
243   * @param regionservers
244   */
245  public static void shutdown(final List<MasterThread> masters,
246      final List<RegionServerThread> regionservers) {
247    LOG.debug("Shutting down HBase Cluster");
248    if (masters != null) {
249      // Do backups first.
250      JVMClusterUtil.MasterThread activeMaster = null;
251      for (JVMClusterUtil.MasterThread t : masters) {
252        // Master was killed but could be still considered as active. Check first if it is stopped.
253        if (!t.master.isStopped()) {
254          if (!t.master.isActiveMaster()) {
255            try {
256              t.master.stopMaster();
257            } catch (IOException e) {
258              LOG.error("Exception occurred while stopping master", e);
259            }
260            LOG.info("Stopped backup Master {} is stopped: {}",
261                t.master.hashCode(), t.master.isStopped());
262          } else {
263            if (activeMaster != null) {
264              LOG.warn("Found more than 1 active master, hash {}", activeMaster.master.hashCode());
265            }
266            activeMaster = t;
267            LOG.debug("Found active master hash={}, stopped={}",
268                t.master.hashCode(), t.master.isStopped());
269          }
270        }
271      }
272      // Do active after.
273      if (activeMaster != null) {
274        try {
275          activeMaster.master.shutdown();
276        } catch (IOException e) {
277          LOG.error("Exception occurred in HMaster.shutdown()", e);
278        }
279      }
280    }
281    boolean wasInterrupted = false;
282    final long maxTime = System.currentTimeMillis() + 30 * 1000;
283    if (regionservers != null) {
284      // first try nicely.
285      for (RegionServerThread t : regionservers) {
286        t.getRegionServer().stop("Shutdown requested");
287      }
288      for (RegionServerThread t : regionservers) {
289        long now = System.currentTimeMillis();
290        if (t.isAlive() && !wasInterrupted && now < maxTime) {
291          try {
292            t.join(maxTime - now);
293          } catch (InterruptedException e) {
294            LOG.info("Got InterruptedException on shutdown - " +
295                "not waiting anymore on region server ends", e);
296            wasInterrupted = true; // someone wants us to speed up.
297          }
298        }
299      }
300
301      // Let's try to interrupt the remaining threads if any.
302      for (int i = 0; i < 100; ++i) {
303        boolean atLeastOneLiveServer = false;
304        for (RegionServerThread t : regionservers) {
305          if (t.isAlive()) {
306            atLeastOneLiveServer = true;
307            try {
308              LOG.warn("RegionServerThreads remaining, give one more chance before interrupting");
309              t.join(1000);
310            } catch (InterruptedException e) {
311              wasInterrupted = true;
312            }
313          }
314        }
315        if (!atLeastOneLiveServer) break;
316        for (RegionServerThread t : regionservers) {
317          if (t.isAlive()) {
318            LOG.warn("RegionServerThreads taking too long to stop, interrupting; thread dump "  +
319              "if > 3 attempts: i=" + i);
320            if (i > 3) {
321              Threads.printThreadInfo(System.out, "Thread dump " + t.getName());
322            }
323            t.interrupt();
324          }
325        }
326      }
327    }
328
329    if (masters != null) {
330      for (JVMClusterUtil.MasterThread t : masters) {
331        while (t.master.isAlive() && !wasInterrupted) {
332          try {
333            // The below has been replaced to debug sometime hangs on end of
334            // tests.
335            // this.master.join():
336            Threads.threadDumpingIsAlive(t.master.getThread());
337          } catch(InterruptedException e) {
338            LOG.info("Got InterruptedException on shutdown - " +
339                "not waiting anymore on master ends", e);
340            wasInterrupted = true;
341          }
342        }
343      }
344    }
345    LOG.info("Shutdown of " +
346      ((masters != null) ? masters.size() : "0") + " master(s) and " +
347      ((regionservers != null) ? regionservers.size() : "0") +
348      " regionserver(s) " + (wasInterrupted ? "interrupted" : "complete"));
349
350    if (wasInterrupted){
351      Thread.currentThread().interrupt();
352    }
353  }
354}