001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.mockito.Mockito.atLeastOnce;
024import static org.mockito.Mockito.never;
025import static org.mockito.Mockito.spy;
026import static org.mockito.Mockito.verify;
027
028import java.util.concurrent.TimeUnit;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.testclassification.MediumTests;
031import org.apache.hadoop.hbase.testclassification.MiscTests;
032import org.apache.hadoop.hbase.util.Threads;
033import org.junit.After;
034import org.junit.Before;
035import org.junit.ClassRule;
036import org.junit.Rule;
037import org.junit.Test;
038import org.junit.experimental.categories.Category;
039import org.junit.rules.TestName;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043@Category({ MiscTests.class, MediumTests.class })
044public class TestChoreService {
045
046  @ClassRule
047  public static final HBaseClassTestRule CLASS_RULE =
048    HBaseClassTestRule.forClass(TestChoreService.class);
049
050  private static final Logger LOG = LoggerFactory.getLogger(TestChoreService.class);
051
052  private static final Configuration CONF = HBaseConfiguration.create();
053
054  @Rule
055  public TestName name = new TestName();
056
057  private int initialCorePoolSize = 3;
058
059  private ChoreService service;
060
061  @Before
062  public void setUp() {
063    service = new ChoreService(name.getMethodName(), initialCorePoolSize, false);
064  }
065
066  @After
067  public void tearDown() {
068    shutdownService(service);
069  }
070
071  /**
072   * Straight forward stopper implementation that is used by default when one is not provided
073   */
074  private static class SampleStopper implements Stoppable {
075    private boolean stopped = false;
076
077    @Override
078    public void stop(String why) {
079      stopped = true;
080    }
081
082    @Override
083    public boolean isStopped() {
084      return stopped;
085    }
086  }
087
088  /**
089   * Sleeps for longer than the scheduled period. This chore always misses its scheduled periodic
090   * executions
091   */
092  private static class SlowChore extends ScheduledChore {
093    public SlowChore(String name, int period) {
094      this(name, new SampleStopper(), period);
095    }
096
097    public SlowChore(String name, Stoppable stopper, int period) {
098      super(name, stopper, period);
099    }
100
101    @Override
102    protected boolean initialChore() {
103      Threads.sleep(getPeriod() * 2);
104      return true;
105    }
106
107    @Override
108    protected void chore() {
109      Threads.sleep(getPeriod() * 2);
110    }
111  }
112
113  /**
114   * Lightweight ScheduledChore used primarily to fill the scheduling queue in tests
115   */
116  private static class DoNothingChore extends ScheduledChore {
117
118    public DoNothingChore(String name, int period) {
119      super(name, new SampleStopper(), period);
120    }
121
122    public DoNothingChore(String name, Stoppable stopper, int period) {
123      super(name, stopper, period);
124    }
125
126    @Override
127    protected void chore() {
128      // DO NOTHING
129    }
130  }
131
132  private static class SleepingChore extends ScheduledChore {
133    private int sleepTime;
134
135    public SleepingChore(String name, int chorePeriod, int sleepTime) {
136      this(name, new SampleStopper(), chorePeriod, sleepTime);
137    }
138
139    public SleepingChore(String name, Stoppable stopper, int period, int sleepTime) {
140      super(name, stopper, period);
141      this.sleepTime = sleepTime;
142    }
143
144    @Override
145    protected boolean initialChore() {
146      Threads.sleep(sleepTime);
147      return true;
148    }
149
150    @Override
151    protected void chore() {
152      Threads.sleep(sleepTime);
153    }
154  }
155
156  private static class CountingChore extends ScheduledChore {
157    private int countOfChoreCalls;
158    private boolean outputOnTicks = false;
159
160    public CountingChore(String name, int period) {
161      this(name, new SampleStopper(), period);
162    }
163
164    public CountingChore(String name, Stoppable stopper, int period) {
165      this(name, stopper, period, false);
166    }
167
168    public CountingChore(String name, Stoppable stopper, int period, final boolean outputOnTicks) {
169      super(name, stopper, period);
170      this.countOfChoreCalls = 0;
171      this.outputOnTicks = outputOnTicks;
172    }
173
174    @Override
175    protected boolean initialChore() {
176      countOfChoreCalls++;
177      if (outputOnTicks) {
178        outputTickCount();
179      }
180      return true;
181    }
182
183    @Override
184    protected void chore() {
185      countOfChoreCalls++;
186      if (outputOnTicks) {
187        outputTickCount();
188      }
189    }
190
191    private void outputTickCount() {
192      LOG.info("Chore: " + getName() + ". Count of chore calls: " + countOfChoreCalls);
193    }
194
195    public int getCountOfChoreCalls() {
196      return countOfChoreCalls;
197    }
198  }
199
200  /**
201   * A Chore that will try to execute the initial chore a few times before succeeding. Once the
202   * initial chore is complete the chore cancels itself
203   */
204  public static class FailInitialChore extends ScheduledChore {
205    private int numberOfFailures;
206    private int failureThreshold;
207
208    /**
209     * @param failThreshold Number of times the Chore fails when trying to execute initialChore
210     *          before succeeding.
211     */
212    public FailInitialChore(String name, int period, int failThreshold) {
213      this(name, new SampleStopper(), period, failThreshold);
214    }
215
216    public FailInitialChore(String name, Stoppable stopper, int period, int failThreshold) {
217      super(name, stopper, period);
218      numberOfFailures = 0;
219      failureThreshold = failThreshold;
220    }
221
222    @Override
223    protected boolean initialChore() {
224      if (numberOfFailures < failureThreshold) {
225        numberOfFailures++;
226        return false;
227      } else {
228        return true;
229      }
230    }
231
232    @Override
233    protected void chore() {
234      assertTrue(numberOfFailures == failureThreshold);
235      cancel(false);
236    }
237  }
238
239  @Test
240  public void testInitialChorePrecedence() throws InterruptedException {
241    final int period = 100;
242    final int failureThreshold = 5;
243    ScheduledChore chore = new FailInitialChore("chore", period, failureThreshold);
244    service.scheduleChore(chore);
245
246    int loopCount = 0;
247    boolean brokeOutOfLoop = false;
248
249    while (!chore.isInitialChoreComplete() && chore.isScheduled()) {
250      Thread.sleep(failureThreshold * period);
251      loopCount++;
252      if (loopCount > 3) {
253        brokeOutOfLoop = true;
254        break;
255      }
256    }
257
258    assertFalse(brokeOutOfLoop);
259  }
260
261  @Test
262  public void testCancelChore() throws InterruptedException {
263    final int period = 100;
264    ScheduledChore chore = new DoNothingChore("chore", period);
265    service.scheduleChore(chore);
266    assertTrue(chore.isScheduled());
267
268    chore.cancel(true);
269    assertFalse(chore.isScheduled());
270    assertTrue(service.getNumberOfScheduledChores() == 0);
271  }
272
273  @Test
274  public void testScheduledChoreConstruction() {
275    final String NAME = "chore";
276    final int PERIOD = 100;
277    final long VALID_DELAY = 0;
278    final long INVALID_DELAY = -100;
279    final TimeUnit UNIT = TimeUnit.NANOSECONDS;
280
281    ScheduledChore chore1 =
282      new ScheduledChore(NAME, new SampleStopper(), PERIOD, VALID_DELAY, UNIT) {
283        @Override
284        protected void chore() {
285          // DO NOTHING
286        }
287      };
288
289    assertEquals("Name construction failed", NAME, chore1.getName());
290    assertEquals("Period construction failed", PERIOD, chore1.getPeriod());
291    assertEquals("Initial Delay construction failed", VALID_DELAY, chore1.getInitialDelay());
292    assertEquals("TimeUnit construction failed", UNIT, chore1.getTimeUnit());
293
294    ScheduledChore invalidDelayChore =
295      new ScheduledChore(NAME, new SampleStopper(), PERIOD, INVALID_DELAY, UNIT) {
296        @Override
297        protected void chore() {
298          // DO NOTHING
299        }
300      };
301
302    assertEquals("Initial Delay should be set to 0 when invalid", 0,
303      invalidDelayChore.getInitialDelay());
304  }
305
306  @Test
307  public void testChoreServiceConstruction() throws InterruptedException {
308    final int corePoolSize = 10;
309    final int defaultCorePoolSize = ChoreService.MIN_CORE_POOL_SIZE;
310
311    ChoreService customInit =
312      new ChoreService("testChoreServiceConstruction_custom", corePoolSize, false);
313    try {
314      assertEquals(corePoolSize, customInit.getCorePoolSize());
315    } finally {
316      shutdownService(customInit);
317    }
318
319    ChoreService defaultInit = new ChoreService("testChoreServiceConstruction_default");
320    try {
321      assertEquals(defaultCorePoolSize, defaultInit.getCorePoolSize());
322    } finally {
323      shutdownService(defaultInit);
324    }
325
326    ChoreService invalidInit = new ChoreService("testChoreServiceConstruction_invalid", -10, false);
327    try {
328      assertEquals(defaultCorePoolSize, invalidInit.getCorePoolSize());
329    } finally {
330      shutdownService(invalidInit);
331    }
332  }
333
334  @Test
335  public void testFrequencyOfChores() throws InterruptedException {
336    final int period = 100;
337    // Small delta that acts as time buffer (allowing chores to complete if running slowly)
338    final int delta = period / 5;
339    CountingChore chore = new CountingChore("countingChore", period);
340    service.scheduleChore(chore);
341
342    Thread.sleep(10 * period + delta);
343    assertEquals("10 periods have elapsed.", 11, chore.getCountOfChoreCalls());
344
345    Thread.sleep(10 * period + delta);
346    assertEquals("20 periods have elapsed.", 21, chore.getCountOfChoreCalls());
347  }
348
349  public void shutdownService(ChoreService service) {
350    service.shutdown();
351    Waiter.waitFor(CONF, 1000, () -> service.isTerminated());
352  }
353
354  @Test
355  public void testForceTrigger() throws InterruptedException {
356    final int period = 100;
357    final int delta = period / 10;
358    final CountingChore chore = new CountingChore("countingChore", period);
359    service.scheduleChore(chore);
360    Thread.sleep(10 * period + delta);
361
362    assertEquals("10 periods have elapsed.", 11, chore.getCountOfChoreCalls());
363
364    // Force five runs of the chore to occur, sleeping between triggers to ensure the
365    // chore has time to run
366    chore.triggerNow();
367    Thread.sleep(delta);
368    chore.triggerNow();
369    Thread.sleep(delta);
370    chore.triggerNow();
371    Thread.sleep(delta);
372    chore.triggerNow();
373    Thread.sleep(delta);
374    chore.triggerNow();
375    Thread.sleep(delta);
376
377    assertEquals("Trigger was called 5 times after 10 periods.", 16, chore.getCountOfChoreCalls());
378
379    Thread.sleep(10 * period + delta);
380
381    // Be loosey-goosey. It used to be '26' but it was a big flakey relying on timing.
382    assertTrue("Expected at least 16 invocations, instead got " + chore.getCountOfChoreCalls(),
383      chore.getCountOfChoreCalls() > 16);
384  }
385
386  @Test
387  public void testCorePoolIncrease() throws InterruptedException {
388    assertEquals("Setting core pool size gave unexpected results.", initialCorePoolSize,
389      service.getCorePoolSize());
390
391    final int slowChorePeriod = 100;
392    SlowChore slowChore1 = new SlowChore("slowChore1", slowChorePeriod);
393    SlowChore slowChore2 = new SlowChore("slowChore2", slowChorePeriod);
394    SlowChore slowChore3 = new SlowChore("slowChore3", slowChorePeriod);
395
396    service.scheduleChore(slowChore1);
397    service.scheduleChore(slowChore2);
398    service.scheduleChore(slowChore3);
399
400    Thread.sleep(slowChorePeriod * 10);
401    assertEquals("Should not create more pools than scheduled chores", 3,
402      service.getCorePoolSize());
403
404    SlowChore slowChore4 = new SlowChore("slowChore4", slowChorePeriod);
405    service.scheduleChore(slowChore4);
406
407    Thread.sleep(slowChorePeriod * 10);
408    assertEquals("Chores are missing their start time. Should expand core pool size", 4,
409      service.getCorePoolSize());
410
411    SlowChore slowChore5 = new SlowChore("slowChore5", slowChorePeriod);
412    service.scheduleChore(slowChore5);
413
414    Thread.sleep(slowChorePeriod * 10);
415    assertEquals("Chores are missing their start time. Should expand core pool size", 5,
416      service.getCorePoolSize());
417  }
418
419  @Test
420  public void testCorePoolDecrease() throws InterruptedException {
421    final int chorePeriod = 100;
422    // Slow chores always miss their start time and thus the core pool size should be at least as
423    // large as the number of running slow chores
424    SlowChore slowChore1 = new SlowChore("slowChore1", chorePeriod);
425    SlowChore slowChore2 = new SlowChore("slowChore2", chorePeriod);
426    SlowChore slowChore3 = new SlowChore("slowChore3", chorePeriod);
427
428    service.scheduleChore(slowChore1);
429    service.scheduleChore(slowChore2);
430    service.scheduleChore(slowChore3);
431
432    Thread.sleep(chorePeriod * 10);
433    assertEquals("Should not create more pools than scheduled chores",
434      service.getNumberOfScheduledChores(), service.getCorePoolSize());
435
436    SlowChore slowChore4 = new SlowChore("slowChore4", chorePeriod);
437    service.scheduleChore(slowChore4);
438    Thread.sleep(chorePeriod * 10);
439    assertEquals("Chores are missing their start time. Should expand core pool size",
440      service.getNumberOfScheduledChores(), service.getCorePoolSize());
441
442    SlowChore slowChore5 = new SlowChore("slowChore5", chorePeriod);
443    service.scheduleChore(slowChore5);
444    Thread.sleep(chorePeriod * 10);
445    assertEquals("Chores are missing their start time. Should expand core pool size",
446      service.getNumberOfScheduledChores(), service.getCorePoolSize());
447    assertEquals(5, service.getNumberOfChoresMissingStartTime());
448
449    // Now we begin to cancel the chores that caused an increase in the core thread pool of the
450    // ChoreService. These cancellations should cause a decrease in the core thread pool.
451    slowChore5.cancel();
452    Thread.sleep(chorePeriod * 10);
453    assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
454      service.getCorePoolSize());
455    assertEquals(4, service.getNumberOfChoresMissingStartTime());
456
457    slowChore4.cancel();
458    Thread.sleep(chorePeriod * 10);
459    assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
460      service.getCorePoolSize());
461    assertEquals(3, service.getNumberOfChoresMissingStartTime());
462
463    slowChore3.cancel();
464    Thread.sleep(chorePeriod * 10);
465    assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
466      service.getCorePoolSize());
467    assertEquals(2, service.getNumberOfChoresMissingStartTime());
468
469    slowChore2.cancel();
470    Thread.sleep(chorePeriod * 10);
471    assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
472      service.getCorePoolSize());
473    assertEquals(1, service.getNumberOfChoresMissingStartTime());
474
475    slowChore1.cancel();
476    Thread.sleep(chorePeriod * 10);
477    assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
478      service.getCorePoolSize());
479    assertEquals(0, service.getNumberOfChoresMissingStartTime());
480  }
481
482  @Test
483  public void testNumberOfRunningChores() throws InterruptedException {
484    final int period = 100;
485    final int sleepTime = 5;
486    DoNothingChore dn1 = new DoNothingChore("dn1", period);
487    DoNothingChore dn2 = new DoNothingChore("dn2", period);
488    DoNothingChore dn3 = new DoNothingChore("dn3", period);
489    DoNothingChore dn4 = new DoNothingChore("dn4", period);
490    DoNothingChore dn5 = new DoNothingChore("dn5", period);
491
492    service.scheduleChore(dn1);
493    service.scheduleChore(dn2);
494    service.scheduleChore(dn3);
495    service.scheduleChore(dn4);
496    service.scheduleChore(dn5);
497
498    Thread.sleep(sleepTime);
499    assertEquals("Scheduled chore mismatch", 5, service.getNumberOfScheduledChores());
500
501    dn1.cancel();
502    Thread.sleep(sleepTime);
503    assertEquals("Scheduled chore mismatch", 4, service.getNumberOfScheduledChores());
504
505    dn2.cancel();
506    dn3.cancel();
507    dn4.cancel();
508    Thread.sleep(sleepTime);
509    assertEquals("Scheduled chore mismatch", 1, service.getNumberOfScheduledChores());
510
511    dn5.cancel();
512    Thread.sleep(sleepTime);
513    assertEquals("Scheduled chore mismatch", 0, service.getNumberOfScheduledChores());
514  }
515
516  @Test
517  public void testNumberOfChoresMissingStartTime() throws InterruptedException {
518    final int period = 100;
519    final int sleepTime = 20 * period;
520    // Slow chores sleep for a length of time LONGER than their period. Thus, SlowChores
521    // ALWAYS miss their start time since their execution takes longer than their period
522    SlowChore sc1 = new SlowChore("sc1", period);
523    SlowChore sc2 = new SlowChore("sc2", period);
524    SlowChore sc3 = new SlowChore("sc3", period);
525    SlowChore sc4 = new SlowChore("sc4", period);
526    SlowChore sc5 = new SlowChore("sc5", period);
527
528    service.scheduleChore(sc1);
529    service.scheduleChore(sc2);
530    service.scheduleChore(sc3);
531    service.scheduleChore(sc4);
532    service.scheduleChore(sc5);
533
534    Thread.sleep(sleepTime);
535    assertEquals(5, service.getNumberOfChoresMissingStartTime());
536
537    sc1.cancel();
538    Thread.sleep(sleepTime);
539    assertEquals(4, service.getNumberOfChoresMissingStartTime());
540
541    sc2.cancel();
542    sc3.cancel();
543    sc4.cancel();
544    Thread.sleep(sleepTime);
545    assertEquals(1, service.getNumberOfChoresMissingStartTime());
546
547    sc5.cancel();
548    Thread.sleep(sleepTime);
549    assertEquals(0, service.getNumberOfChoresMissingStartTime());
550  }
551
552  /**
553   * ChoreServices should never have a core pool size that exceeds the number of chores that have
554   * been scheduled with the service. For example, if 4 ScheduledChores are scheduled with a
555   * ChoreService, the number of threads in the ChoreService's core pool should never exceed 4
556   */
557  @Test
558  public void testMaximumChoreServiceThreads() throws InterruptedException {
559
560    final int period = 100;
561    final int sleepTime = 5 * period;
562    // Slow chores sleep for a length of time LONGER than their period. Thus, SlowChores
563    // ALWAYS miss their start time since their execution takes longer than their period.
564    // Chores that miss their start time will trigger the onChoreMissedStartTime callback
565    // in the ChoreService. This callback will try to increase the number of core pool
566    // threads.
567    SlowChore sc1 = new SlowChore("sc1", period);
568    SlowChore sc2 = new SlowChore("sc2", period);
569    SlowChore sc3 = new SlowChore("sc3", period);
570    SlowChore sc4 = new SlowChore("sc4", period);
571    SlowChore sc5 = new SlowChore("sc5", period);
572
573    service.scheduleChore(sc1);
574    service.scheduleChore(sc2);
575    service.scheduleChore(sc3);
576    service.scheduleChore(sc4);
577    service.scheduleChore(sc5);
578
579    Thread.sleep(sleepTime);
580    assertTrue(service.getCorePoolSize() <= service.getNumberOfScheduledChores());
581
582    SlowChore sc6 = new SlowChore("sc6", period);
583    SlowChore sc7 = new SlowChore("sc7", period);
584    SlowChore sc8 = new SlowChore("sc8", period);
585    SlowChore sc9 = new SlowChore("sc9", period);
586    SlowChore sc10 = new SlowChore("sc10", period);
587
588    service.scheduleChore(sc6);
589    service.scheduleChore(sc7);
590    service.scheduleChore(sc8);
591    service.scheduleChore(sc9);
592    service.scheduleChore(sc10);
593
594    Thread.sleep(sleepTime);
595    assertTrue(service.getCorePoolSize() <= service.getNumberOfScheduledChores());
596  }
597
598  @Test
599  public void testChangingChoreServices() throws InterruptedException {
600    final int period = 100;
601    final int sleepTime = 10;
602    ChoreService anotherService = new ChoreService(name.getMethodName() + "_2");
603    ScheduledChore chore = new DoNothingChore("sample", period);
604
605    try {
606      assertFalse(chore.isScheduled());
607      assertFalse(service.isChoreScheduled(chore));
608      assertFalse(anotherService.isChoreScheduled(chore));
609      assertTrue(chore.getChoreService() == null);
610
611      service.scheduleChore(chore);
612      Thread.sleep(sleepTime);
613      assertTrue(chore.isScheduled());
614      assertTrue(service.isChoreScheduled(chore));
615      assertFalse(anotherService.isChoreScheduled(chore));
616      assertFalse(chore.getChoreService() == null);
617
618      anotherService.scheduleChore(chore);
619      Thread.sleep(sleepTime);
620      assertTrue(chore.isScheduled());
621      assertFalse(service.isChoreScheduled(chore));
622      assertTrue(anotherService.isChoreScheduled(chore));
623      assertFalse(chore.getChoreService() == null);
624
625      chore.cancel();
626      assertFalse(chore.isScheduled());
627      assertFalse(service.isChoreScheduled(chore));
628      assertFalse(anotherService.isChoreScheduled(chore));
629      assertTrue(chore.getChoreService() == null);
630    } finally {
631      shutdownService(anotherService);
632    }
633  }
634
635  @Test
636  public void testStopperForScheduledChores() throws InterruptedException {
637    Stoppable stopperForGroup1 = new SampleStopper();
638    Stoppable stopperForGroup2 = new SampleStopper();
639    final int period = 100;
640    final int delta = period / 10;
641    ScheduledChore chore1_group1 = new DoNothingChore("c1g1", stopperForGroup1, period);
642    ScheduledChore chore2_group1 = new DoNothingChore("c2g1", stopperForGroup1, period);
643    ScheduledChore chore3_group1 = new DoNothingChore("c3g1", stopperForGroup1, period);
644
645    ScheduledChore chore1_group2 = new DoNothingChore("c1g2", stopperForGroup2, period);
646    ScheduledChore chore2_group2 = new DoNothingChore("c2g2", stopperForGroup2, period);
647    ScheduledChore chore3_group2 = new DoNothingChore("c3g2", stopperForGroup2, period);
648
649    service.scheduleChore(chore1_group1);
650    service.scheduleChore(chore2_group1);
651    service.scheduleChore(chore3_group1);
652    service.scheduleChore(chore1_group2);
653    service.scheduleChore(chore2_group2);
654    service.scheduleChore(chore3_group2);
655
656    Thread.sleep(delta);
657    Thread.sleep(10 * period);
658    assertTrue(chore1_group1.isScheduled());
659    assertTrue(chore2_group1.isScheduled());
660    assertTrue(chore3_group1.isScheduled());
661    assertTrue(chore1_group2.isScheduled());
662    assertTrue(chore2_group2.isScheduled());
663    assertTrue(chore3_group2.isScheduled());
664
665    stopperForGroup1.stop("test stopping group 1");
666    Thread.sleep(period);
667    assertFalse(chore1_group1.isScheduled());
668    assertFalse(chore2_group1.isScheduled());
669    assertFalse(chore3_group1.isScheduled());
670    assertTrue(chore1_group2.isScheduled());
671    assertTrue(chore2_group2.isScheduled());
672    assertTrue(chore3_group2.isScheduled());
673
674    stopperForGroup2.stop("test stopping group 2");
675    Thread.sleep(period);
676    assertFalse(chore1_group1.isScheduled());
677    assertFalse(chore2_group1.isScheduled());
678    assertFalse(chore3_group1.isScheduled());
679    assertFalse(chore1_group2.isScheduled());
680    assertFalse(chore2_group2.isScheduled());
681    assertFalse(chore3_group2.isScheduled());
682  }
683
684  @Test
685  public void testShutdownCancelsScheduledChores() throws InterruptedException {
686    final int period = 100;
687    ScheduledChore successChore1 = new DoNothingChore("sc1", period);
688    ScheduledChore successChore2 = new DoNothingChore("sc2", period);
689    ScheduledChore successChore3 = new DoNothingChore("sc3", period);
690    assertTrue(service.scheduleChore(successChore1));
691    assertTrue(successChore1.isScheduled());
692    assertTrue(service.scheduleChore(successChore2));
693    assertTrue(successChore2.isScheduled());
694    assertTrue(service.scheduleChore(successChore3));
695    assertTrue(successChore3.isScheduled());
696
697    shutdownService(service);
698
699    assertFalse(successChore1.isScheduled());
700    assertFalse(successChore2.isScheduled());
701    assertFalse(successChore3.isScheduled());
702  }
703
704  @Test
705  public void testShutdownWorksWhileChoresAreExecuting() throws InterruptedException {
706    final int period = 100;
707    final int sleep = 5 * period;
708    ScheduledChore slowChore1 = new SleepingChore("sc1", period, sleep);
709    ScheduledChore slowChore2 = new SleepingChore("sc2", period, sleep);
710    ScheduledChore slowChore3 = new SleepingChore("sc3", period, sleep);
711    assertTrue(service.scheduleChore(slowChore1));
712    assertTrue(service.scheduleChore(slowChore2));
713    assertTrue(service.scheduleChore(slowChore3));
714
715    Thread.sleep(sleep / 2);
716    shutdownService(service);
717
718    assertFalse(slowChore1.isScheduled());
719    assertFalse(slowChore2.isScheduled());
720    assertFalse(slowChore3.isScheduled());
721    assertTrue(service.isShutdown());
722
723    Thread.sleep(5);
724    assertTrue(service.isTerminated());
725  }
726
727  @Test
728  public void testShutdownRejectsNewSchedules() throws InterruptedException {
729    final int period = 100;
730    ScheduledChore successChore1 = new DoNothingChore("sc1", period);
731    ScheduledChore successChore2 = new DoNothingChore("sc2", period);
732    ScheduledChore successChore3 = new DoNothingChore("sc3", period);
733    ScheduledChore failChore1 = new DoNothingChore("fc1", period);
734    ScheduledChore failChore2 = new DoNothingChore("fc2", period);
735    ScheduledChore failChore3 = new DoNothingChore("fc3", period);
736
737    assertTrue(service.scheduleChore(successChore1));
738    assertTrue(successChore1.isScheduled());
739    assertTrue(service.scheduleChore(successChore2));
740    assertTrue(successChore2.isScheduled());
741    assertTrue(service.scheduleChore(successChore3));
742    assertTrue(successChore3.isScheduled());
743
744    shutdownService(service);
745
746    assertFalse(service.scheduleChore(failChore1));
747    assertFalse(failChore1.isScheduled());
748    assertFalse(service.scheduleChore(failChore2));
749    assertFalse(failChore2.isScheduled());
750    assertFalse(service.scheduleChore(failChore3));
751    assertFalse(failChore3.isScheduled());
752  }
753
754  /**
755   * for HBASE-25014
756   */
757  @Test
758  public void testInitialDelay() {
759    SampleStopper stopper = new SampleStopper();
760    service.scheduleChore(new ScheduledChore("chore", stopper, 1000, 2000) {
761      @Override
762      protected void chore() {
763        stopper.stop("test");
764      }
765    });
766    Waiter.waitFor(CONF, 5000, () -> stopper.isStopped());
767  }
768
769  @Test
770  public void testCleanupWithStopper() {
771    SampleStopper stopper = new SampleStopper();
772    DoNothingChore chore = spy(new DoNothingChore("chore", stopper, 10));
773    service.scheduleChore(chore);
774    assertTrue(chore.isScheduled());
775    verify(chore, never()).cleanup();
776    stopper.stop("test");
777    Waiter.waitFor(CONF, 200, () -> !chore.isScheduled());
778    verify(chore, atLeastOnce()).cleanup();
779  }
780
781  @Test
782  public void testCleanupWithShutdown() {
783    DoNothingChore chore = spy(new DoNothingChore("chore", 10));
784    service.scheduleChore(chore);
785    assertTrue(chore.isScheduled());
786    verify(chore, never()).cleanup();
787    chore.shutdown(true);
788    Waiter.waitFor(CONF, 200, () -> !chore.isScheduled());
789    verify(chore, atLeastOnce()).cleanup();
790  }
791}