001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.tool; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNotEquals; 023import static org.junit.Assert.assertNotNull; 024import static org.junit.Assert.assertTrue; 025import static org.mockito.ArgumentMatchers.argThat; 026import static org.mockito.Matchers.anyLong; 027import static org.mockito.Matchers.eq; 028import static org.mockito.Matchers.isA; 029import static org.mockito.Mockito.atLeastOnce; 030import static org.mockito.Mockito.never; 031import static org.mockito.Mockito.spy; 032import static org.mockito.Mockito.times; 033import static org.mockito.Mockito.verify; 034 035import java.util.List; 036import java.util.Map; 037import java.util.concurrent.ExecutorService; 038import java.util.concurrent.ScheduledThreadPoolExecutor; 039import org.apache.hadoop.hbase.HBaseClassTestRule; 040import org.apache.hadoop.hbase.HBaseTestingUtility; 041import org.apache.hadoop.hbase.HConstants; 042import org.apache.hadoop.hbase.ServerName; 043import org.apache.hadoop.hbase.TableName; 044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 045import org.apache.hadoop.hbase.client.Put; 046import org.apache.hadoop.hbase.client.RegionInfo; 047import org.apache.hadoop.hbase.client.Table; 048import org.apache.hadoop.hbase.testclassification.MediumTests; 049import org.apache.hadoop.hbase.util.Bytes; 050import org.apache.hadoop.util.ToolRunner; 051import org.apache.log4j.Appender; 052import org.apache.log4j.LogManager; 053import org.apache.log4j.spi.LoggingEvent; 054import org.junit.After; 055import org.junit.Before; 056import org.junit.ClassRule; 057import org.junit.Rule; 058import org.junit.Test; 059import org.junit.experimental.categories.Category; 060import org.junit.rules.TestName; 061import org.junit.runner.RunWith; 062import org.mockito.ArgumentMatcher; 063import org.mockito.Mock; 064import org.mockito.runners.MockitoJUnitRunner; 065 066import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; 067 068@RunWith(MockitoJUnitRunner.class) 069@Category({MediumTests.class}) 070public class TestCanaryTool { 071 072 @ClassRule 073 public static final HBaseClassTestRule CLASS_RULE = 074 HBaseClassTestRule.forClass(TestCanaryTool.class); 075 076 private HBaseTestingUtility testingUtility; 077 private static final byte[] FAMILY = Bytes.toBytes("f"); 078 private static final byte[] COLUMN = Bytes.toBytes("col"); 079 080 @Rule 081 public TestName name = new TestName(); 082 083 @Before 084 public void setUp() throws Exception { 085 testingUtility = new HBaseTestingUtility(); 086 testingUtility.startMiniCluster(); 087 LogManager.getRootLogger().addAppender(mockAppender); 088 } 089 090 @After 091 public void tearDown() throws Exception { 092 testingUtility.shutdownMiniCluster(); 093 LogManager.getRootLogger().removeAppender(mockAppender); 094 } 095 096 @Mock 097 Appender mockAppender; 098 099 @Test 100 public void testBasicZookeeperCanaryWorks() throws Exception { 101 final String[] args = { "-t", "10000", "-zookeeper" }; 102 testZookeeperCanaryWithArgs(args); 103 } 104 105 @Test 106 public void testZookeeperCanaryPermittedFailuresArgumentWorks() throws Exception { 107 final String[] args = { "-t", "10000", "-zookeeper", "-treatFailureAsError", "-permittedZookeeperFailures", "1" }; 108 testZookeeperCanaryWithArgs(args); 109 } 110 111 @Test 112 public void testBasicCanaryWorks() throws Exception { 113 final TableName tableName = TableName.valueOf(name.getMethodName()); 114 Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY }); 115 // insert some test rows 116 for (int i=0; i<1000; i++) { 117 byte[] iBytes = Bytes.toBytes(i); 118 Put p = new Put(iBytes); 119 p.addColumn(FAMILY, COLUMN, iBytes); 120 table.put(p); 121 } 122 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 123 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 124 CanaryTool canary = new CanaryTool(executor, sink); 125 String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() }; 126 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 127 assertEquals("verify no read error count", 0, canary.getReadFailures().size()); 128 assertEquals("verify no write error count", 0, canary.getWriteFailures().size()); 129 verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class), 130 isA(ColumnFamilyDescriptor.class), anyLong()); 131 } 132 133 @Test 134 public void testCanaryRegionTaskResult() throws Exception { 135 TableName tableName = TableName.valueOf("testCanaryRegionTaskResult"); 136 Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY }); 137 // insert some test rows 138 for (int i=0; i<1000; i++) { 139 byte[] iBytes = Bytes.toBytes(i); 140 Put p = new Put(iBytes); 141 p.addColumn(FAMILY, COLUMN, iBytes); 142 table.put(p); 143 } 144 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 145 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 146 CanaryTool canary = new CanaryTool(executor, sink); 147 String[] args = { "-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult" }; 148 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 149 150 assertTrue("canary should expect to scan at least 1 region", 151 sink.getTotalExpectedRegions() > 0); 152 assertTrue("there should be no read failures", sink.getReadFailureCount() == 0); 153 assertTrue("there should be no write failures", sink.getWriteFailureCount() == 0); 154 assertTrue("verify read success count > 0", sink.getReadSuccessCount() > 0); 155 assertTrue("verify write success count > 0", sink.getWriteSuccessCount() > 0); 156 verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class), 157 isA(ColumnFamilyDescriptor.class), anyLong()); 158 verify(sink, atLeastOnce()).publishWriteTiming(isA(ServerName.class), isA(RegionInfo.class), 159 isA(ColumnFamilyDescriptor.class), anyLong()); 160 161 assertEquals("canary region success count should equal total expected regions", 162 sink.getReadSuccessCount() + sink.getWriteSuccessCount(), sink.getTotalExpectedRegions()); 163 Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap(); 164 assertFalse("verify region map has size > 0", regionMap.isEmpty()); 165 166 for (String regionName : regionMap.keySet()) { 167 for (CanaryTool.RegionTaskResult res: regionMap.get(regionName)) { 168 assertNotNull("verify getRegionNameAsString()", regionName); 169 assertNotNull("verify getRegionInfo()", res.getRegionInfo()); 170 assertNotNull("verify getTableName()", res.getTableName()); 171 assertNotNull("verify getTableNameAsString()", res.getTableNameAsString()); 172 assertNotNull("verify getServerName()", res.getServerName()); 173 assertNotNull("verify getServerNameAsString()", res.getServerNameAsString()); 174 assertNotNull("verify getColumnFamily()", res.getColumnFamily()); 175 assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString()); 176 177 if (regionName.contains(CanaryTool.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) { 178 assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess()); 179 assertTrue("write took some time", res.getWriteLatency() > -1); 180 } else { 181 assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess()); 182 assertTrue("read took some time", res.getReadLatency() > -1); 183 } 184 } 185 } 186 } 187 188 @Test 189 public void testReadTableTimeouts() throws Exception { 190 final TableName [] tableNames = new TableName[2]; 191 tableNames[0] = TableName.valueOf(name.getMethodName() + "1"); 192 tableNames[1] = TableName.valueOf(name.getMethodName() + "2"); 193 // Create 2 test tables. 194 for (int j = 0; j<2; j++) { 195 Table table = testingUtility.createTable(tableNames[j], new byte[][] { FAMILY }); 196 // insert some test rows 197 for (int i=0; i<1000; i++) { 198 byte[] iBytes = Bytes.toBytes(i + j); 199 Put p = new Put(iBytes); 200 p.addColumn(FAMILY, COLUMN, iBytes); 201 table.put(p); 202 } 203 } 204 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 205 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 206 CanaryTool canary = new CanaryTool(executor, sink); 207 String configuredTimeoutStr = tableNames[0].getNameAsString() + "=" + Long.MAX_VALUE + "," + 208 tableNames[1].getNameAsString() + "=0"; 209 String[] args = {"-readTableTimeouts", configuredTimeoutStr, name.getMethodName() + "1", 210 name.getMethodName() + "2"}; 211 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 212 verify(sink, times(tableNames.length)).initializeAndGetReadLatencyForTable(isA(String.class)); 213 for (int i=0; i<2; i++) { 214 assertNotEquals("verify non-null read latency", null, sink.getReadLatencyMap().get(tableNames[i].getNameAsString())); 215 assertNotEquals("verify non-zero read latency", 0L, sink.getReadLatencyMap().get(tableNames[i].getNameAsString())); 216 } 217 // One table's timeout is set for 0 ms and thus, should lead to an error. 218 verify(mockAppender, times(1)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() { 219 @Override 220 public boolean matches(LoggingEvent argument) { 221 return ((LoggingEvent) argument).getRenderedMessage().contains("exceeded the configured read timeout."); 222 } 223 })); 224 verify(mockAppender, times(2)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() { 225 @Override 226 public boolean matches(LoggingEvent argument) { 227 return argument.getRenderedMessage().contains("Configured read timeout"); 228 } 229 })); 230 } 231 232 @Test 233 public void testWriteTableTimeout() throws Exception { 234 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 235 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 236 CanaryTool canary = new CanaryTool(executor, sink); 237 String[] args = { "-writeSniffing", "-writeTableTimeout", String.valueOf(Long.MAX_VALUE)}; 238 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 239 assertNotEquals("verify non-null write latency", null, sink.getWriteLatency()); 240 assertNotEquals("verify non-zero write latency", 0L, sink.getWriteLatency()); 241 verify(mockAppender, times(1)).doAppend(argThat( 242 new ArgumentMatcher<LoggingEvent>() { 243 @Override 244 public boolean matches(LoggingEvent argument) { 245 return argument.getRenderedMessage().contains("Configured write timeout"); 246 } 247 })); 248 } 249 250 //no table created, so there should be no regions 251 @Test 252 public void testRegionserverNoRegions() throws Exception { 253 runRegionserverCanary(); 254 verify(mockAppender).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() { 255 @Override 256 public boolean matches(LoggingEvent argument) { 257 return argument.getRenderedMessage().contains("Regionserver not serving any regions"); 258 } 259 })); 260 } 261 262 //by creating a table, there shouldn't be any region servers not serving any regions 263 @Test 264 public void testRegionserverWithRegions() throws Exception { 265 final TableName tableName = TableName.valueOf(name.getMethodName()); 266 testingUtility.createTable(tableName, new byte[][] { FAMILY }); 267 runRegionserverCanary(); 268 verify(mockAppender, never()).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() { 269 @Override 270 public boolean matches(LoggingEvent argument) { 271 return argument.getRenderedMessage().contains("Regionserver not serving any regions"); 272 } 273 })); 274 } 275 276 @Test 277 public void testRawScanConfig() throws Exception { 278 final TableName tableName = TableName.valueOf(name.getMethodName()); 279 Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY }); 280 // insert some test rows 281 for (int i=0; i<1000; i++) { 282 byte[] iBytes = Bytes.toBytes(i); 283 Put p = new Put(iBytes); 284 p.addColumn(FAMILY, COLUMN, iBytes); 285 table.put(p); 286 } 287 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 288 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 289 CanaryTool canary = new CanaryTool(executor, sink); 290 String[] args = { "-t", "10000", name.getMethodName() }; 291 org.apache.hadoop.conf.Configuration conf = 292 new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration()); 293 conf.setBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, true); 294 assertEquals(0, ToolRunner.run(conf, canary, args)); 295 verify(sink, atLeastOnce()) 296 .publishReadTiming(isA(ServerName.class), isA(RegionInfo.class), 297 isA(ColumnFamilyDescriptor.class), anyLong()); 298 assertEquals("verify no read error count", 0, canary.getReadFailures().size()); 299 } 300 301 private void runRegionserverCanary() throws Exception { 302 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 303 CanaryTool canary = new CanaryTool(executor, new CanaryTool.RegionServerStdOutSink()); 304 String[] args = { "-t", "10000", "-regionserver"}; 305 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 306 assertEquals("verify no read error count", 0, canary.getReadFailures().size()); 307 } 308 309 private void testZookeeperCanaryWithArgs(String[] args) throws Exception { 310 Integer port = 311 Iterables.getOnlyElement(testingUtility.getZkCluster().getClientPortList(), null); 312 testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM, 313 "localhost:" + port + "/hbase"); 314 ExecutorService executor = new ScheduledThreadPoolExecutor(2); 315 CanaryTool.ZookeeperStdOutSink sink = spy(new CanaryTool.ZookeeperStdOutSink()); 316 CanaryTool canary = new CanaryTool(executor, sink); 317 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 318 319 String baseZnode = testingUtility.getConfiguration() 320 .get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); 321 verify(sink, atLeastOnce()) 322 .publishReadTiming(eq(baseZnode), eq("localhost:" + port), anyLong()); 323 } 324}