001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.tool;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertTrue;
025import static org.mockito.ArgumentMatchers.anyLong;
026import static org.mockito.ArgumentMatchers.argThat;
027import static org.mockito.ArgumentMatchers.eq;
028import static org.mockito.ArgumentMatchers.isA;
029import static org.mockito.Mockito.atLeastOnce;
030import static org.mockito.Mockito.mock;
031import static org.mockito.Mockito.never;
032import static org.mockito.Mockito.spy;
033import static org.mockito.Mockito.times;
034import static org.mockito.Mockito.verify;
035import static org.mockito.Mockito.when;
036
037import java.util.List;
038import java.util.Map;
039import java.util.concurrent.ExecutorService;
040import java.util.concurrent.ScheduledThreadPoolExecutor;
041import org.apache.hadoop.conf.Configuration;
042import org.apache.hadoop.hbase.HBaseClassTestRule;
043import org.apache.hadoop.hbase.HBaseConfiguration;
044import org.apache.hadoop.hbase.HBaseTestingUtility;
045import org.apache.hadoop.hbase.HConstants;
046import org.apache.hadoop.hbase.ServerName;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
049import org.apache.hadoop.hbase.client.Put;
050import org.apache.hadoop.hbase.client.RegionInfo;
051import org.apache.hadoop.hbase.client.Table;
052import org.apache.hadoop.hbase.testclassification.LargeTests;
053import org.apache.hadoop.hbase.util.Bytes;
054import org.apache.hadoop.util.ToolRunner;
055import org.junit.After;
056import org.junit.Before;
057import org.junit.ClassRule;
058import org.junit.Rule;
059import org.junit.Test;
060import org.junit.experimental.categories.Category;
061import org.junit.rules.TestName;
062import org.mockito.ArgumentMatcher;
063
064@Category({ LargeTests.class })
065public class TestCanaryTool {
066
067  @ClassRule
068  public static final HBaseClassTestRule CLASS_RULE =
069    HBaseClassTestRule.forClass(TestCanaryTool.class);
070
071  private HBaseTestingUtility testingUtility;
072  private static final byte[] FAMILY = Bytes.toBytes("f");
073  private static final byte[] COLUMN = Bytes.toBytes("col");
074
075  @Rule
076  public TestName name = new TestName();
077
078  private org.apache.logging.log4j.core.Appender mockAppender;
079
080  @Before
081  public void setUp() throws Exception {
082    testingUtility = new HBaseTestingUtility();
083    testingUtility.startMiniCluster();
084    mockAppender = mock(org.apache.logging.log4j.core.Appender.class);
085    when(mockAppender.getName()).thenReturn("mockAppender");
086    when(mockAppender.isStarted()).thenReturn(true);
087    ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager
088      .getLogger("org.apache.hadoop.hbase")).addAppender(mockAppender);
089  }
090
091  @After
092  public void tearDown() throws Exception {
093    testingUtility.shutdownMiniCluster();
094    ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager
095      .getLogger("org.apache.hadoop.hbase")).removeAppender(mockAppender);
096  }
097
098  @Test
099  public void testBasicZookeeperCanaryWorks() throws Exception {
100    final String[] args = { "-t", "10000", "-zookeeper" };
101    testZookeeperCanaryWithArgs(args);
102  }
103
104  @Test
105  public void testZookeeperCanaryPermittedFailuresArgumentWorks() throws Exception {
106    final String[] args =
107      { "-t", "10000", "-zookeeper", "-treatFailureAsError", "-permittedZookeeperFailures", "1" };
108    testZookeeperCanaryWithArgs(args);
109  }
110
111  @Test
112  public void testBasicCanaryWorks() throws Exception {
113    final TableName tableName = TableName.valueOf(name.getMethodName());
114    Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
115    // insert some test rows
116    for (int i = 0; i < 1000; i++) {
117      byte[] iBytes = Bytes.toBytes(i);
118      Put p = new Put(iBytes);
119      p.addColumn(FAMILY, COLUMN, iBytes);
120      table.put(p);
121    }
122    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
123    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
124    CanaryTool canary = new CanaryTool(executor, sink);
125    String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() };
126    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
127    assertEquals("verify no read error count", 0, canary.getReadFailures().size());
128    assertEquals("verify no write error count", 0, canary.getWriteFailures().size());
129    verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
130      isA(ColumnFamilyDescriptor.class), anyLong());
131  }
132
133  @Test
134  public void testCanaryRegionTaskReadAllCF() throws Exception {
135    final TableName tableName = TableName.valueOf(name.getMethodName());
136    Table table = testingUtility.createTable(tableName,
137      new byte[][] { Bytes.toBytes("f1"), Bytes.toBytes("f2") });
138    // insert some test rows
139    for (int i = 0; i < 1000; i++) {
140      byte[] iBytes = Bytes.toBytes(i);
141      Put p = new Put(iBytes);
142      p.addColumn(Bytes.toBytes("f1"), COLUMN, iBytes);
143      p.addColumn(Bytes.toBytes("f2"), COLUMN, iBytes);
144      table.put(p);
145    }
146    Configuration configuration = HBaseConfiguration.create(testingUtility.getConfiguration());
147    String[] args = { "-t", "10000", "testCanaryRegionTaskReadAllCF" };
148    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
149    for (boolean readAllCF : new boolean[] { true, false }) {
150      CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
151      CanaryTool canary = new CanaryTool(executor, sink);
152      configuration.setBoolean(HConstants.HBASE_CANARY_READ_ALL_CF, readAllCF);
153      assertEquals(0, ToolRunner.run(configuration, canary, args));
154      // the test table has two column family. If readAllCF set true,
155      // we expect read count is double of region count
156      int expectedReadCount =
157        readAllCF ? 2 * sink.getTotalExpectedRegions() : sink.getTotalExpectedRegions();
158      assertEquals("canary region success count should equal total expected read count",
159        expectedReadCount, sink.getReadSuccessCount());
160      Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap();
161      assertFalse("verify region map has size > 0", regionMap.isEmpty());
162
163      for (String regionName : regionMap.keySet()) {
164        for (CanaryTool.RegionTaskResult res : regionMap.get(regionName)) {
165          assertNotNull("verify getRegionNameAsString()", regionName);
166          assertNotNull("verify getRegionInfo()", res.getRegionInfo());
167          assertNotNull("verify getTableName()", res.getTableName());
168          assertNotNull("verify getTableNameAsString()", res.getTableNameAsString());
169          assertNotNull("verify getServerName()", res.getServerName());
170          assertNotNull("verify getServerNameAsString()", res.getServerNameAsString());
171          assertNotNull("verify getColumnFamily()", res.getColumnFamily());
172          assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString());
173          assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess());
174          assertTrue("read took some time", res.getReadLatency() > -1);
175        }
176      }
177    }
178  }
179
180  @Test
181  public void testCanaryRegionTaskResult() throws Exception {
182    TableName tableName = TableName.valueOf("testCanaryRegionTaskResult");
183    Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
184    // insert some test rows
185    for (int i = 0; i < 1000; i++) {
186      byte[] iBytes = Bytes.toBytes(i);
187      Put p = new Put(iBytes);
188      p.addColumn(FAMILY, COLUMN, iBytes);
189      table.put(p);
190    }
191    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
192    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
193    CanaryTool canary = new CanaryTool(executor, sink);
194    String[] args = { "-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult" };
195    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
196
197    assertTrue("canary should expect to scan at least 1 region",
198      sink.getTotalExpectedRegions() > 0);
199    assertTrue("there should be no read failures", sink.getReadFailureCount() == 0);
200    assertTrue("there should be no write failures", sink.getWriteFailureCount() == 0);
201    assertTrue("verify read success count > 0", sink.getReadSuccessCount() > 0);
202    assertTrue("verify write success count > 0", sink.getWriteSuccessCount() > 0);
203    verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
204      isA(ColumnFamilyDescriptor.class), anyLong());
205    verify(sink, atLeastOnce()).publishWriteTiming(isA(ServerName.class), isA(RegionInfo.class),
206      isA(ColumnFamilyDescriptor.class), anyLong());
207
208    assertEquals("canary region success count should equal total expected regions",
209      sink.getReadSuccessCount() + sink.getWriteSuccessCount(), sink.getTotalExpectedRegions());
210    Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap();
211    assertFalse("verify region map has size > 0", regionMap.isEmpty());
212
213    for (String regionName : regionMap.keySet()) {
214      for (CanaryTool.RegionTaskResult res : regionMap.get(regionName)) {
215        assertNotNull("verify getRegionNameAsString()", regionName);
216        assertNotNull("verify getRegionInfo()", res.getRegionInfo());
217        assertNotNull("verify getTableName()", res.getTableName());
218        assertNotNull("verify getTableNameAsString()", res.getTableNameAsString());
219        assertNotNull("verify getServerName()", res.getServerName());
220        assertNotNull("verify getServerNameAsString()", res.getServerNameAsString());
221        assertNotNull("verify getColumnFamily()", res.getColumnFamily());
222        assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString());
223
224        if (regionName.contains(CanaryTool.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) {
225          assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess());
226          assertTrue("write took some time", res.getWriteLatency() > -1);
227        } else {
228          assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess());
229          assertTrue("read took some time", res.getReadLatency() > -1);
230        }
231      }
232    }
233  }
234
235  // Ignore this test. It fails w/ the below on some mac os x.
236  // [ERROR] Failures:
237  // [ERROR] TestCanaryTool.testReadTableTimeouts:216
238  // Argument(s) are different! Wanted:
239  // mockAppender.doAppend(
240  // <custom argument matcher>
241  // );
242  // -> at org.apache.hadoop.hbase.tool.TestCanaryTool
243  // .testReadTableTimeouts(TestCanaryTool.java:216)
244  // Actual invocations have different arguments:
245  // mockAppender.doAppend(
246  // org.apache.log4j.spi.LoggingEvent@2055cfc1
247  // );
248  // )
249  // )
250  //
251  @org.junit.Ignore
252  @Test
253  public void testReadTableTimeouts() throws Exception {
254    final TableName[] tableNames = new TableName[] { TableName.valueOf(name.getMethodName() + "1"),
255      TableName.valueOf(name.getMethodName() + "2") };
256    // Create 2 test tables.
257    for (int j = 0; j < 2; j++) {
258      Table table = testingUtility.createTable(tableNames[j], new byte[][] { FAMILY });
259      // insert some test rows
260      for (int i = 0; i < 10; i++) {
261        byte[] iBytes = Bytes.toBytes(i + j);
262        Put p = new Put(iBytes);
263        p.addColumn(FAMILY, COLUMN, iBytes);
264        table.put(p);
265      }
266    }
267    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
268    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
269    CanaryTool canary = new CanaryTool(executor, sink);
270    String configuredTimeoutStr = tableNames[0].getNameAsString() + "=" + Long.MAX_VALUE + ","
271      + tableNames[1].getNameAsString() + "=0";
272    String[] args = { "-readTableTimeouts", configuredTimeoutStr, name.getMethodName() + "1",
273      name.getMethodName() + "2" };
274    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
275    verify(sink, times(tableNames.length)).initializeAndGetReadLatencyForTable(isA(String.class));
276    for (int i = 0; i < 2; i++) {
277      assertNotEquals("verify non-null read latency", null,
278        sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
279      assertNotEquals("verify non-zero read latency", 0L,
280        sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
281    }
282    // One table's timeout is set for 0 ms and thus, should lead to an error.
283    verify(mockAppender, times(1))
284      .append(argThat(new ArgumentMatcher<org.apache.logging.log4j.core.LogEvent>() {
285        @Override
286        public boolean matches(org.apache.logging.log4j.core.LogEvent argument) {
287          return argument.getMessage().getFormattedMessage()
288            .contains("exceeded the configured read timeout.");
289        }
290      }));
291    verify(mockAppender, times(2))
292      .append(argThat(new ArgumentMatcher<org.apache.logging.log4j.core.LogEvent>() {
293        @Override
294        public boolean matches(org.apache.logging.log4j.core.LogEvent argument) {
295          return argument.getMessage().getFormattedMessage().contains("Configured read timeout");
296        }
297      }));
298  }
299
300  @Test
301  public void testWriteTableTimeout() throws Exception {
302    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
303    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
304    CanaryTool canary = new CanaryTool(executor, sink);
305    String[] args = { "-writeSniffing", "-writeTableTimeout", String.valueOf(Long.MAX_VALUE) };
306    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
307    assertNotEquals("verify non-null write latency", null, sink.getWriteLatency());
308    assertNotEquals("verify non-zero write latency", 0L, sink.getWriteLatency());
309    verify(mockAppender, times(1))
310      .append(argThat(new ArgumentMatcher<org.apache.logging.log4j.core.LogEvent>() {
311        @Override
312        public boolean matches(org.apache.logging.log4j.core.LogEvent argument) {
313          return argument.getMessage().getFormattedMessage().contains("Configured write timeout");
314        }
315      }));
316  }
317
318  // no table created, so there should be no regions
319  @Test
320  public void testRegionserverNoRegions() throws Exception {
321    runRegionserverCanary();
322    verify(mockAppender)
323      .append(argThat(new ArgumentMatcher<org.apache.logging.log4j.core.LogEvent>() {
324        @Override
325        public boolean matches(org.apache.logging.log4j.core.LogEvent argument) {
326          return argument.getMessage().getFormattedMessage()
327            .contains("Regionserver not serving any regions");
328        }
329      }));
330  }
331
332  // by creating a table, there shouldn't be any region servers not serving any regions
333  @Test
334  public void testRegionserverWithRegions() throws Exception {
335    final TableName tableName = TableName.valueOf(name.getMethodName());
336    testingUtility.createTable(tableName, new byte[][] { FAMILY });
337    runRegionserverCanary();
338    verify(mockAppender, never())
339      .append(argThat(new ArgumentMatcher<org.apache.logging.log4j.core.LogEvent>() {
340        @Override
341        public boolean matches(org.apache.logging.log4j.core.LogEvent argument) {
342          return argument.getMessage().getFormattedMessage()
343            .contains("Regionserver not serving any regions");
344        }
345      }));
346  }
347
348  @Test
349  public void testRawScanConfig() throws Exception {
350    final TableName tableName = TableName.valueOf(name.getMethodName());
351    Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
352    // insert some test rows
353    for (int i = 0; i < 1000; i++) {
354      byte[] iBytes = Bytes.toBytes(i);
355      Put p = new Put(iBytes);
356      p.addColumn(FAMILY, COLUMN, iBytes);
357      table.put(p);
358    }
359    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
360    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
361    CanaryTool canary = new CanaryTool(executor, sink);
362    String[] args = { "-t", "10000", name.getMethodName() };
363    org.apache.hadoop.conf.Configuration conf =
364      new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration());
365    conf.setBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, true);
366    assertEquals(0, ToolRunner.run(conf, canary, args));
367    verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
368      isA(ColumnFamilyDescriptor.class), anyLong());
369    assertEquals("verify no read error count", 0, canary.getReadFailures().size());
370  }
371
372  private void runRegionserverCanary() throws Exception {
373    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
374    CanaryTool canary = new CanaryTool(executor, new CanaryTool.RegionServerStdOutSink());
375    String[] args = { "-t", "10000", "-regionserver" };
376    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
377    assertEquals("verify no read error count", 0, canary.getReadFailures().size());
378  }
379
380  private void testZookeeperCanaryWithArgs(String[] args) throws Exception {
381    String hostPort = testingUtility.getZkCluster().getAddress().toString();
382    testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM, hostPort + "/hbase");
383    ExecutorService executor = new ScheduledThreadPoolExecutor(2);
384    CanaryTool.ZookeeperStdOutSink sink = spy(new CanaryTool.ZookeeperStdOutSink());
385    CanaryTool canary = new CanaryTool(executor, sink);
386    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
387
388    String baseZnode = testingUtility.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT,
389      HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
390    verify(sink, atLeastOnce()).publishReadTiming(eq(baseZnode), eq(hostPort), anyLong());
391  }
392}