001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.tool;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertTrue;
025import static org.mockito.ArgumentMatchers.argThat;
026import static org.mockito.Matchers.anyLong;
027import static org.mockito.Matchers.eq;
028import static org.mockito.Matchers.isA;
029import static org.mockito.Mockito.atLeastOnce;
030import static org.mockito.Mockito.never;
031import static org.mockito.Mockito.spy;
032import static org.mockito.Mockito.times;
033import static org.mockito.Mockito.verify;
034
035import java.util.List;
036import java.util.Map;
037import java.util.concurrent.ExecutorService;
038import java.util.concurrent.ScheduledThreadPoolExecutor;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtility;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.ServerName;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
045import org.apache.hadoop.hbase.client.Put;
046import org.apache.hadoop.hbase.client.RegionInfo;
047import org.apache.hadoop.hbase.client.Table;
048import org.apache.hadoop.hbase.testclassification.MediumTests;
049import org.apache.hadoop.hbase.util.Bytes;
050import org.apache.hadoop.util.ToolRunner;
051import org.apache.log4j.Appender;
052import org.apache.log4j.LogManager;
053import org.apache.log4j.spi.LoggingEvent;
054import org.junit.After;
055import org.junit.Before;
056import org.junit.ClassRule;
057import org.junit.Rule;
058import org.junit.Test;
059import org.junit.experimental.categories.Category;
060import org.junit.rules.TestName;
061import org.junit.runner.RunWith;
062import org.mockito.ArgumentMatcher;
063import org.mockito.Mock;
064import org.mockito.runners.MockitoJUnitRunner;
065
066import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
067
068@RunWith(MockitoJUnitRunner.class)
069@Category({MediumTests.class})
070public class TestCanaryTool {
071
072  @ClassRule
073  public static final HBaseClassTestRule CLASS_RULE =
074      HBaseClassTestRule.forClass(TestCanaryTool.class);
075
076  private HBaseTestingUtility testingUtility;
077  private static final byte[] FAMILY = Bytes.toBytes("f");
078  private static final byte[] COLUMN = Bytes.toBytes("col");
079
080  @Rule
081  public TestName name = new TestName();
082
083  @Before
084  public void setUp() throws Exception {
085    testingUtility = new HBaseTestingUtility();
086    testingUtility.startMiniCluster();
087    LogManager.getRootLogger().addAppender(mockAppender);
088  }
089
090  @After
091  public void tearDown() throws Exception {
092    testingUtility.shutdownMiniCluster();
093    LogManager.getRootLogger().removeAppender(mockAppender);
094  }
095
096  @Mock
097  Appender mockAppender;
098
099  @Test
100  public void testBasicZookeeperCanaryWorks() throws Exception {
101    final String[] args = { "-t", "10000", "-zookeeper" };
102    testZookeeperCanaryWithArgs(args);
103  }
104
105  @Test
106  public void testZookeeperCanaryPermittedFailuresArgumentWorks() throws Exception {
107    final String[] args = { "-t", "10000", "-zookeeper", "-treatFailureAsError", "-permittedZookeeperFailures", "1" };
108    testZookeeperCanaryWithArgs(args);
109  }
110
111  @Test
112  public void testBasicCanaryWorks() throws Exception {
113    final TableName tableName = TableName.valueOf(name.getMethodName());
114    Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
115    // insert some test rows
116    for (int i=0; i<1000; i++) {
117      byte[] iBytes = Bytes.toBytes(i);
118      Put p = new Put(iBytes);
119      p.addColumn(FAMILY, COLUMN, iBytes);
120      table.put(p);
121    }
122    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
123    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
124    CanaryTool canary = new CanaryTool(executor, sink);
125    String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() };
126    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
127    assertEquals("verify no read error count", 0, canary.getReadFailures().size());
128    assertEquals("verify no write error count", 0, canary.getWriteFailures().size());
129    verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
130      isA(ColumnFamilyDescriptor.class), anyLong());
131  }
132
133  @Test
134  public void testCanaryRegionTaskResult() throws Exception {
135    TableName tableName = TableName.valueOf("testCanaryRegionTaskResult");
136    Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
137    // insert some test rows
138    for (int i=0; i<1000; i++) {
139      byte[] iBytes = Bytes.toBytes(i);
140      Put p = new Put(iBytes);
141      p.addColumn(FAMILY, COLUMN, iBytes);
142      table.put(p);
143    }
144    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
145    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
146    CanaryTool canary = new CanaryTool(executor, sink);
147    String[] args = { "-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult" };
148    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
149
150    assertTrue("canary should expect to scan at least 1 region",
151      sink.getTotalExpectedRegions() > 0);
152    assertTrue("there should be no read failures", sink.getReadFailureCount() == 0);
153    assertTrue("there should be no write failures", sink.getWriteFailureCount() == 0);
154    assertTrue("verify read success count > 0", sink.getReadSuccessCount() > 0);
155    assertTrue("verify write success count > 0", sink.getWriteSuccessCount() > 0);
156    verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
157      isA(ColumnFamilyDescriptor.class), anyLong());
158    verify(sink, atLeastOnce()).publishWriteTiming(isA(ServerName.class), isA(RegionInfo.class),
159      isA(ColumnFamilyDescriptor.class), anyLong());
160
161    assertEquals("canary region success count should equal total expected regions",
162      sink.getReadSuccessCount() + sink.getWriteSuccessCount(), sink.getTotalExpectedRegions());
163    Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap();
164    assertFalse("verify region map has size > 0", regionMap.isEmpty());
165
166    for (String regionName : regionMap.keySet()) {
167      for (CanaryTool.RegionTaskResult res: regionMap.get(regionName)) {
168        assertNotNull("verify getRegionNameAsString()", regionName);
169        assertNotNull("verify getRegionInfo()", res.getRegionInfo());
170        assertNotNull("verify getTableName()", res.getTableName());
171        assertNotNull("verify getTableNameAsString()", res.getTableNameAsString());
172        assertNotNull("verify getServerName()", res.getServerName());
173        assertNotNull("verify getServerNameAsString()", res.getServerNameAsString());
174        assertNotNull("verify getColumnFamily()", res.getColumnFamily());
175        assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString());
176
177        if (regionName.contains(CanaryTool.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) {
178          assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess());
179          assertTrue("write took some time", res.getWriteLatency() > -1);
180        } else {
181          assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess());
182          assertTrue("read took some time", res.getReadLatency() > -1);
183        }
184      }
185    }
186  }
187
188  @Test
189  public void testReadTableTimeouts() throws Exception {
190    final TableName [] tableNames = new TableName[2];
191    tableNames[0] = TableName.valueOf(name.getMethodName() + "1");
192    tableNames[1] = TableName.valueOf(name.getMethodName() + "2");
193    // Create 2 test tables.
194    for (int j = 0; j<2; j++) {
195      Table table = testingUtility.createTable(tableNames[j], new byte[][] { FAMILY });
196      // insert some test rows
197      for (int i=0; i<1000; i++) {
198        byte[] iBytes = Bytes.toBytes(i + j);
199        Put p = new Put(iBytes);
200        p.addColumn(FAMILY, COLUMN, iBytes);
201        table.put(p);
202      }
203    }
204    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
205    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
206    CanaryTool canary = new CanaryTool(executor, sink);
207    String configuredTimeoutStr = tableNames[0].getNameAsString() + "=" + Long.MAX_VALUE + "," +
208      tableNames[1].getNameAsString() + "=0";
209    String[] args = {"-readTableTimeouts", configuredTimeoutStr, name.getMethodName() + "1",
210      name.getMethodName() + "2"};
211    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
212    verify(sink, times(tableNames.length)).initializeAndGetReadLatencyForTable(isA(String.class));
213    for (int i=0; i<2; i++) {
214      assertNotEquals("verify non-null read latency", null, sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
215      assertNotEquals("verify non-zero read latency", 0L, sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
216    }
217    // One table's timeout is set for 0 ms and thus, should lead to an error.
218    verify(mockAppender, times(1)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
219      @Override
220      public boolean matches(LoggingEvent argument) {
221        return ((LoggingEvent) argument).getRenderedMessage().contains("exceeded the configured read timeout.");
222      }
223    }));
224    verify(mockAppender, times(2)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
225      @Override
226      public boolean matches(LoggingEvent argument) {
227        return argument.getRenderedMessage().contains("Configured read timeout");
228      }
229    }));
230  }
231
232  @Test
233  public void testWriteTableTimeout() throws Exception {
234    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
235    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
236    CanaryTool canary = new CanaryTool(executor, sink);
237    String[] args = { "-writeSniffing", "-writeTableTimeout", String.valueOf(Long.MAX_VALUE)};
238    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
239    assertNotEquals("verify non-null write latency", null, sink.getWriteLatency());
240    assertNotEquals("verify non-zero write latency", 0L, sink.getWriteLatency());
241    verify(mockAppender, times(1)).doAppend(argThat(
242        new ArgumentMatcher<LoggingEvent>() {
243          @Override
244          public boolean matches(LoggingEvent argument) {
245            return argument.getRenderedMessage().contains("Configured write timeout");
246          }
247        }));
248  }
249
250  //no table created, so there should be no regions
251  @Test
252  public void testRegionserverNoRegions() throws Exception {
253    runRegionserverCanary();
254    verify(mockAppender).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
255      @Override
256      public boolean matches(LoggingEvent argument) {
257        return argument.getRenderedMessage().contains("Regionserver not serving any regions");
258      }
259    }));
260  }
261
262  //by creating a table, there shouldn't be any region servers not serving any regions
263  @Test
264  public void testRegionserverWithRegions() throws Exception {
265    final TableName tableName = TableName.valueOf(name.getMethodName());
266    testingUtility.createTable(tableName, new byte[][] { FAMILY });
267    runRegionserverCanary();
268    verify(mockAppender, never()).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
269      @Override
270      public boolean matches(LoggingEvent argument) {
271        return argument.getRenderedMessage().contains("Regionserver not serving any regions");
272      }
273    }));
274  }
275
276  @Test
277  public void testRawScanConfig() throws Exception {
278    final TableName tableName = TableName.valueOf(name.getMethodName());
279    Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
280    // insert some test rows
281    for (int i=0; i<1000; i++) {
282      byte[] iBytes = Bytes.toBytes(i);
283      Put p = new Put(iBytes);
284      p.addColumn(FAMILY, COLUMN, iBytes);
285      table.put(p);
286    }
287    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
288    CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
289    CanaryTool canary = new CanaryTool(executor, sink);
290    String[] args = { "-t", "10000", name.getMethodName() };
291    org.apache.hadoop.conf.Configuration conf =
292      new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration());
293    conf.setBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, true);
294    assertEquals(0, ToolRunner.run(conf, canary, args));
295    verify(sink, atLeastOnce())
296        .publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
297        isA(ColumnFamilyDescriptor.class), anyLong());
298    assertEquals("verify no read error count", 0, canary.getReadFailures().size());
299  }
300
301  private void runRegionserverCanary() throws Exception {
302    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
303    CanaryTool canary = new CanaryTool(executor, new CanaryTool.RegionServerStdOutSink());
304    String[] args = { "-t", "10000", "-regionserver"};
305    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
306    assertEquals("verify no read error count", 0, canary.getReadFailures().size());
307  }
308
309  private void testZookeeperCanaryWithArgs(String[] args) throws Exception {
310    Integer port =
311      Iterables.getOnlyElement(testingUtility.getZkCluster().getClientPortList(), null);
312    testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM,
313      "localhost:" + port + "/hbase");
314    ExecutorService executor = new ScheduledThreadPoolExecutor(2);
315    CanaryTool.ZookeeperStdOutSink sink = spy(new CanaryTool.ZookeeperStdOutSink());
316    CanaryTool canary = new CanaryTool(executor, sink);
317    assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
318
319    String baseZnode = testingUtility.getConfiguration()
320      .get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
321    verify(sink, atLeastOnce())
322      .publishReadTiming(eq(baseZnode), eq("localhost:" + port), anyLong());
323  }
324}