/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.tool;

import static org.apache.hadoop.hbase.regionserver.TestRegionServerNoMaster.closeRegion;
import static org.apache.hadoop.hbase.tool.CanaryTool.HBASE_CANARY_INFO_PORT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.LongAdder;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JvmVersion;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.mockito.ArgumentMatcher;

@Category({ LargeTests.class })
public class TestCanaryTool {

  @ClassRule
  public static final HBaseClassTestRule CLASS_RULE =
    HBaseClassTestRule.forClass(TestCanaryTool.class);

  private HBaseTestingUtil testingUtility;
  private static final byte[] FAMILY = Bytes.toBytes("f");
  private static final byte[] COLUMN = Bytes.toBytes("col");

  @Rule
  public TestName name = new TestName();

  private org.apache.logging.log4j.core.Appender mockAppender;

  @Before
  public void setUp() throws Exception {
    testingUtility = new HBaseTestingUtil();
    testingUtility.startMiniCluster();
    mockAppender = mock(org.apache.logging.log4j.core.Appender.class);
    when(mockAppender.getName()).thenReturn("mockAppender");
    when(mockAppender.isStarted()).thenReturn(true);
    ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager
      .getLogger("org.apache.hadoop.hbase")).addAppender(mockAppender);
  }

  @After
  public void tearDown() throws Exception {
    testingUtility.shutdownMiniCluster();
    ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager
      .getLogger("org.apache.hadoop.hbase")).removeAppender(mockAppender);
  }

  @Test
  public void testBasicZookeeperCanaryWorks() throws Exception {
    final String[] args = { "-t", "10000", "-zookeeper" };
    testZookeeperCanaryWithArgs(args);
  }

  @Test
  public void testZookeeperCanaryPermittedFailuresArgumentWorks() throws Exception {
    final String[] args =
      { "-t", "10000", "-zookeeper", "-treatFailureAsError", "-permittedZookeeperFailures", "1" };
    testZookeeperCanaryWithArgs(args);
  }

  @Test
  public void testBasicCanaryWorks() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
    // insert some test rows
    for (int i = 0; i < 1000; i++) {
      byte[] iBytes = Bytes.toBytes(i);
      Put p = new Put(iBytes);
      p.addColumn(FAMILY, COLUMN, iBytes);
      table.put(p);
    }
    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
    CanaryTool canary = new CanaryTool(executor, sink);
    String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() };
    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
    assertEquals("verify no read error count", 0, canary.getReadFailures().size());
    assertEquals("verify no write error count", 0, canary.getWriteFailures().size());
    verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
      isA(ColumnFamilyDescriptor.class), anyLong());
  }

  /**
   * When CanaryTool times out, it should stop scanning and shutdown quickly and gracefully. This
   * test helps to confirm that threadpools do not continue executing work after the canary
   * finishes. It also verifies sink behavior and measures correct failure counts in the sink.
   * @throws Exception if it can't create a table, communicate with minicluster, or run the canary.
   */
  @Test
  public void testCanaryStopsScanningAfterTimeout() throws Exception {
    // Prepare a table with multiple regions, and close those regions on the regionserver.
    // Do not notify HMaster or META. CanaryTool will scan and receive NotServingRegionExceptions.
    final TableName tableName = TableName.valueOf(name.getMethodName());
    // Close the unused Table reference returned by createMultiRegionTable.
    testingUtility.createMultiRegionTable(tableName, new byte[][] { FAMILY }).close();
    List<RegionInfo> regions = testingUtility.getAdmin().getRegions(tableName);
    assertTrue("verify table has multiple regions", regions.size() > 1);
    HRegionServer regionserver = testingUtility.getMiniHBaseCluster().getRegionServer(0);
    for (RegionInfo region : regions) {
      closeRegion(testingUtility, regionserver, region);
    }

    // Run CanaryTool with 1 thread. This thread will attempt to scan the first region.
    // It will use default rpc retries and receive NotServingRegionExceptions for many seconds
    // according to HConstants.RETRY_BACKOFF. The CanaryTool timeout is set to 4 seconds, so it
    // will time out before the first region scan is complete.
    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
    CanaryTool canary = new CanaryTool(executor);
    String[] args = { "-t", "4000", tableName.getNameAsString() };
    int retCode = ToolRunner.run(testingUtility.getConfiguration(), canary, args);
    executor.shutdown();
    try {
      if (!executor.awaitTermination(3, TimeUnit.SECONDS)) {
        executor.shutdownNow();
      }
    } catch (InterruptedException e) {
      executor.shutdownNow();
    }

    CanaryTool.Sink sink = canary.getActiveSink();
    assertEquals("verify canary timed out with TIMEOUT_ERROR_EXIT_CODE", 3, retCode);
    assertEquals("verify only the first region failed", 1, sink.getReadFailureCount());
    assertEquals("verify no successful reads", 0, sink.getReadSuccessCount());
    assertEquals("verify we were attempting to scan all regions", regions.size(),
      ((CanaryTool.RegionStdOutSink) sink).getTotalExpectedRegions());
  }

  @Test
  public void testCanaryRegionTaskReadAllCF() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    Table table = testingUtility.createTable(tableName,
      new byte[][] { Bytes.toBytes("f1"), Bytes.toBytes("f2") });
    // insert some test rows
    for (int i = 0; i < 1000; i++) {
      byte[] iBytes = Bytes.toBytes(i);
      Put p = new Put(iBytes);
      p.addColumn(Bytes.toBytes("f1"), COLUMN, iBytes);
      p.addColumn(Bytes.toBytes("f2"), COLUMN, iBytes);
      table.put(p);
    }
    Configuration configuration = HBaseConfiguration.create(testingUtility.getConfiguration());
    String[] args = { "-t", "10000", "testCanaryRegionTaskReadAllCF" };
    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
    for (boolean readAllCF : new boolean[] { true, false }) {
      CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
      CanaryTool canary = new CanaryTool(executor, sink);
      configuration.setBoolean(HConstants.HBASE_CANARY_READ_ALL_CF, readAllCF);
      assertEquals(0, ToolRunner.run(configuration, canary, args));
      // the test table has two column family. If readAllCF set true,
      // we expect read count is double of region count
      int expectedReadCount =
        readAllCF ? 2 * sink.getTotalExpectedRegions() : sink.getTotalExpectedRegions();
      assertEquals("canary region success count should equal total expected read count",
        expectedReadCount, sink.getReadSuccessCount());
      Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap();
      assertFalse("verify region map has size > 0", regionMap.isEmpty());

      for (String regionName : regionMap.keySet()) {
        for (CanaryTool.RegionTaskResult res : regionMap.get(regionName)) {
          assertNotNull("verify getRegionNameAsString()", regionName);
          assertNotNull("verify getRegionInfo()", res.getRegionInfo());
          assertNotNull("verify getTableName()", res.getTableName());
          assertNotNull("verify getTableNameAsString()", res.getTableNameAsString());
          assertNotNull("verify getServerName()", res.getServerName());
          assertNotNull("verify getServerNameAsString()", res.getServerNameAsString());
          assertNotNull("verify getColumnFamily()", res.getColumnFamily());
          assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString());
          assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess());
          assertTrue("read took some time", res.getReadLatency() > -1);
        }
      }
    }
  }

  @Test
  public void testCanaryRegionTaskResult() throws Exception {
    TableName tableName = TableName.valueOf("testCanaryRegionTaskResult");
    Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
    // insert some test rows
    for (int i = 0; i < 1000; i++) {
      byte[] iBytes = Bytes.toBytes(i);
      Put p = new Put(iBytes);
      p.addColumn(FAMILY, COLUMN, iBytes);
      table.put(p);
    }
    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
    CanaryTool canary = new CanaryTool(executor, sink);
    String[] args = { "-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult" };
    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));

    assertTrue("canary should expect to scan at least 1 region",
      sink.getTotalExpectedRegions() > 0);
    assertTrue("there should be no read failures", sink.getReadFailureCount() == 0);
    assertTrue("there should be no write failures", sink.getWriteFailureCount() == 0);
    assertTrue("verify read success count > 0", sink.getReadSuccessCount() > 0);
    assertTrue("verify write success count > 0", sink.getWriteSuccessCount() > 0);
    verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
      isA(ColumnFamilyDescriptor.class), anyLong());
    verify(sink, atLeastOnce()).publishWriteTiming(isA(ServerName.class), isA(RegionInfo.class),
      isA(ColumnFamilyDescriptor.class), anyLong());

    assertEquals("canary region success count should equal total expected regions",
      sink.getReadSuccessCount() + sink.getWriteSuccessCount(), sink.getTotalExpectedRegions());
    Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap();
    assertFalse("verify region map has size > 0", regionMap.isEmpty());

    for (String regionName : regionMap.keySet()) {
      for (CanaryTool.RegionTaskResult res : regionMap.get(regionName)) {
        assertNotNull("verify getRegionNameAsString()", regionName);
        assertNotNull("verify getRegionInfo()", res.getRegionInfo());
        assertNotNull("verify getTableName()", res.getTableName());
        assertNotNull("verify getTableNameAsString()", res.getTableNameAsString());
        assertNotNull("verify getServerName()", res.getServerName());
        assertNotNull("verify getServerNameAsString()", res.getServerNameAsString());
        assertNotNull("verify getColumnFamily()", res.getColumnFamily());
        assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString());

        if (regionName.contains(CanaryTool.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) {
          assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess());
          assertTrue("write took some time", res.getWriteLatency() > -1);
        } else {
          assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess());
          assertTrue("read took some time", res.getReadLatency() > -1);
        }
      }
    }
  }

  // Ignore this test. It fails w/ the below on some mac os x.
  // [ERROR] Failures:
  // [ERROR] TestCanaryTool.testReadTableTimeouts:216
  // Argument(s) are different! Wanted:
  // mockAppender.doAppend(
  // <custom argument matcher>
  // );
  // -> at org.apache.hadoop.hbase.tool.TestCanaryTool
  // .testReadTableTimeouts(TestCanaryTool.java:216)
  // Actual invocations have different arguments:
  // mockAppender.doAppend(
  // org.apache.log4j.spi.LoggingEvent@2055cfc1
  // );
  // )
  // )
  //
  @org.junit.Ignore
  @Test
  public void testReadTableTimeouts() throws Exception {
    final TableName[] tableNames = new TableName[] { TableName.valueOf(name.getMethodName() + "1"),
      TableName.valueOf(name.getMethodName() + "2") };
    // Create 2 test tables.
    for (int j = 0; j < 2; j++) {
      Table table = testingUtility.createTable(tableNames[j], new byte[][] { FAMILY });
      // insert some test rows
      for (int i = 0; i < 10; i++) {
        byte[] iBytes = Bytes.toBytes(i + j);
        Put p = new Put(iBytes);
        p.addColumn(FAMILY, COLUMN, iBytes);
        table.put(p);
      }
    }
    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
    CanaryTool canary = new CanaryTool(executor, sink);
    String configuredTimeoutStr = tableNames[0].getNameAsString() + "=" + Long.MAX_VALUE + ","
      + tableNames[1].getNameAsString() + "=0";
    String[] args = { "-readTableTimeouts", configuredTimeoutStr, name.getMethodName() + "1",
      name.getMethodName() + "2" };
    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
    verify(sink, times(tableNames.length)).initializeAndGetReadLatencyForTable(isA(String.class));
    for (int i = 0; i < 2; i++) {
      assertNotEquals("verify non-null read latency", null,
        sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
      assertNotEquals("verify non-zero read latency", 0L,
        sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
    }
    // One table's timeout is set for 0 ms and thus, should lead to an error.
    verify(mockAppender, times(1))
      .append(argThat(new ArgumentMatcher<org.apache.logging.log4j.core.LogEvent>() {
        @Override
        public boolean matches(org.apache.logging.log4j.core.LogEvent argument) {
          return argument.getMessage().getFormattedMessage()
            .contains("exceeded the configured read timeout.");
        }
      }));
    verify(mockAppender, times(2))
      .append(argThat(new ArgumentMatcher<org.apache.logging.log4j.core.LogEvent>() {
        @Override
        public boolean matches(org.apache.logging.log4j.core.LogEvent argument) {
          return argument.getMessage().getFormattedMessage().contains("Configured read timeout");
        }
      }));
  }

  @Test
  public void testWriteTableTimeout() throws Exception {
    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
    CanaryTool canary = new CanaryTool(executor, sink);
    String[] args = { "-writeSniffing", "-writeTableTimeout", String.valueOf(Long.MAX_VALUE) };
    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
    assertNotEquals("verify non-null write latency", null, sink.getWriteLatency());
    assertNotEquals("verify non-zero write latency", 0L, sink.getWriteLatency());
    verify(mockAppender, times(1))
      .append(argThat(new ArgumentMatcher<org.apache.logging.log4j.core.LogEvent>() {
        @Override
        public boolean matches(org.apache.logging.log4j.core.LogEvent argument) {
          return argument.getMessage().getFormattedMessage().contains("Configured write timeout");
        }
      }));
  }

  // no table created, so there should be no regions
  @Test
  public void testRegionserverNoRegions() throws Exception {
    runRegionserverCanary();
    verify(mockAppender)
      .append(argThat(new ArgumentMatcher<org.apache.logging.log4j.core.LogEvent>() {
        @Override
        public boolean matches(org.apache.logging.log4j.core.LogEvent argument) {
          return argument.getMessage().getFormattedMessage()
            .contains("Regionserver not serving any regions");
        }
      }));
  }

  // by creating a table, there shouldn't be any region servers not serving any regions
  @Test
  public void testRegionserverWithRegions() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    testingUtility.createTable(tableName, new byte[][] { FAMILY });
    runRegionserverCanary();
    verify(mockAppender, never())
      .append(argThat(new ArgumentMatcher<org.apache.logging.log4j.core.LogEvent>() {
        @Override
        public boolean matches(org.apache.logging.log4j.core.LogEvent argument) {
          return argument.getMessage().getFormattedMessage()
            .contains("Regionserver not serving any regions");
        }
      }));
  }

  @Test
  public void testRawScanConfig() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
    // insert some test rows
    for (int i = 0; i < 1000; i++) {
      byte[] iBytes = Bytes.toBytes(i);
      Put p = new Put(iBytes);
      p.addColumn(FAMILY, COLUMN, iBytes);
      table.put(p);
    }
    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
    CanaryTool canary = new CanaryTool(executor, sink);
    String[] args = { "-t", "10000", name.getMethodName() };
    org.apache.hadoop.conf.Configuration conf =
      new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration());
    conf.setBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, true);
    assertEquals(0, ToolRunner.run(conf, canary, args));
    verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
      isA(ColumnFamilyDescriptor.class), anyLong());
    assertEquals("verify no read error count", 0, canary.getReadFailures().size());
  }

  private void runRegionserverCanary() throws Exception {
    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
    CanaryTool canary = new CanaryTool(executor, new CanaryTool.RegionServerStdOutSink());
    String[] args = { "-t", "10000", "-regionserver" };
    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
    assertEquals("verify no read error count", 0, canary.getReadFailures().size());
  }

  private void testZookeeperCanaryWithArgs(String[] args) throws Exception {
    String hostPort = testingUtility.getZkCluster().getAddress().toString();
    testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM, hostPort);
    ExecutorService executor = new ScheduledThreadPoolExecutor(2);
    CanaryTool.ZookeeperStdOutSink sink = spy(new CanaryTool.ZookeeperStdOutSink());
    CanaryTool canary = new CanaryTool(executor, sink);
    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));

    String baseZnode = testingUtility.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT,
      HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
    verify(sink, atLeastOnce()).publishReadTiming(eq(baseZnode), eq(hostPort), anyLong());
  }

  @Test
  public void testWebUI() throws Exception {
    CanaryTool.RegionStdOutSink sink = mock(CanaryTool.RegionStdOutSink.class);

    Configuration configuration = HBaseConfiguration.create(testingUtility.getConfiguration());
    int infoPort = 16666;
    configuration.setInt(HBASE_CANARY_INFO_PORT, infoPort);

    ExecutorService executorService = startCanaryToolInBackground(sink, configuration);

    // Test that old canary status page URL redirects to JSP
    URL oldPageUrl = new URL("http://localhost:" + infoPort + "/canary-status");
    String oldPageContent = getPageContent(oldPageUrl);
    assertTrue("expected=canary.jsp, content=" + oldPageContent,
      oldPageContent.contains("canary.jsp"));

    // Test web UI page content
    URL url = new URL("http://localhost:" + infoPort + "/canary.jsp");
    String page = getPageContent(url);

    assertTrue("Page should contain page title.", page.contains("<title>Canary</title>"));

    assertTrue("Page should contain Failed Servers header.",
      page.contains("<h2>Failed Servers</h2>"));
    assertTrue("Page should have zero Failed Servers.",
      page.contains("<td>Total Failed Servers: 0</td>"));

    assertTrue("Page should contain Failed Tables header.",
      page.contains("<h2>Failed Tables</h2>"));
    assertTrue("Page should have zero Failed Tables.",
      page.contains("<td>Total Failed Tables: 0</td>"));

    assertTrue("Page should contain Software Attributes header.",
      page.contains("<h2>Software Attributes</h2>"));
    assertTrue("Page should contain JVM version.",
      page.contains("<td>" + JvmVersion.getVersion() + "</td>"));
    assertTrue("Page should contain HBase version.", page
      .contains("<td>" + VersionInfo.getVersion() + ", r" + VersionInfo.getRevision() + "</td>"));

    // Stop Canary tool daemon
    executorService.shutdown();
  }

  @Test
  public void testWebUIWithFailures() throws Exception {
    CanaryTool.RegionStdOutSink sink = mock(CanaryTool.RegionStdOutSink.class);

    // Simulate a failed server
    ServerName sn1 = ServerName.parseServerName("asf903.gq1.ygridcore.net,52690,1517835491385");
    ConcurrentMap<ServerName, LongAdder> servers = new ConcurrentHashMap<>();
    servers.put(sn1, new LongAdder());
    when(sink.getPerServerFailuresCount()).thenReturn(servers);

    // Simulate failed tables
    ConcurrentMap<String, LongAdder> tables = new ConcurrentHashMap<>();
    tables.put("awesome-table", new LongAdder());
    tables.put("awesome-table-two", new LongAdder());
    when(sink.getPerTableFailuresCount()).thenReturn(tables);

    Configuration configuration = HBaseConfiguration.create(testingUtility.getConfiguration());
    int infoPort = 16667;
    configuration.setInt(HBASE_CANARY_INFO_PORT, infoPort);

    ExecutorService executorService = startCanaryToolInBackground(sink, configuration);

    URL url = new URL("http://localhost:" + infoPort + "/canary.jsp");
    String page = getPageContent(url);

    assertTrue("Page should contain page title.", page.contains("<title>Canary</title>"));

    assertTrue("Page should contain Failed Servers header.",
      page.contains("<h2>Failed Servers</h2>"));
    assertTrue("Page should contain the failed server link.", page.contains(
      "<a href=\"//asf903.gq1.ygridcore.net:52691/\">asf903.gq1.ygridcore.net,52690,1517835491385</a>"));
    assertTrue("Page should summarize 1 failed server.",
      page.contains("<td>Total Failed Servers: 1</td>"));

    assertTrue("Page should contain Failed Tables header.",
      page.contains("<h2>Failed Tables</h2>"));
    assertTrue("Page should contain awesome-table as failed table link.",
      page.contains("<td>awesome-table</td>"));
    assertTrue("Page should contain awesome-table-two as failed table link.",
      page.contains("<td>awesome-table-two</td>"));
    assertTrue("Page should summarize 2 failed tables.",
      page.contains("<td>Total Failed Tables: 2</td>"));

    // Stop Canary tool daemon
    executorService.shutdown();
  }

  private static ExecutorService startCanaryToolInBackground(CanaryTool.RegionStdOutSink sink,
    Configuration configuration) {
    ExecutorService canaryExecutor = new ScheduledThreadPoolExecutor(1);
    CanaryTool canary = new CanaryTool(canaryExecutor, sink);
    String[] args = { "-daemon", "-interval", "5", "-f", "false" };

    // Run the Canary CLI tool in another thread otherwise it would block the unit test thread
    // and we could not examine the web UI page.
    ExecutorService executorService = new ScheduledThreadPoolExecutor(1);
    executorService.submit(() -> ToolRunner.run(configuration, canary, args));
    return executorService;
  }

  private String getPageContent(URL url) throws IOException, InterruptedException {
    HttpURLConnection conn = (HttpURLConnection) url.openConnection();

    boolean success = false;
    for (int i = 1; i <= 5; i++) {
      try {
        conn.connect();
        if (
          conn.getResponseCode() == 200 && "text/html;charset=utf-8".equals(conn.getContentType())
        ) {
          success = true;
          break;
        }
      } catch (IOException e) {
        // ignore connection error as we retry.
      }

      // Wait a bit for the Canary web UI to come up
      TimeUnit.MILLISECONDS.sleep(100);
    }

    if (success) {
      try (InputStream in = conn.getInputStream()) {
        return IOUtils.toString(in, StandardCharsets.UTF_8);
      }
    } else {
      throw new IllegalStateException("Could not get Canary status page.");
    }
  }
}