001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.tool; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNotEquals; 023import static org.junit.Assert.assertNotNull; 024import static org.junit.Assert.assertTrue; 025import static org.mockito.ArgumentMatchers.anyLong; 026import static org.mockito.ArgumentMatchers.argThat; 027import static org.mockito.ArgumentMatchers.eq; 028import static org.mockito.ArgumentMatchers.isA; 029import static org.mockito.Mockito.atLeastOnce; 030import static org.mockito.Mockito.never; 031import static org.mockito.Mockito.spy; 032import static org.mockito.Mockito.times; 033import static org.mockito.Mockito.verify; 034import java.util.List; 035import java.util.Map; 036import java.util.concurrent.ExecutorService; 037import java.util.concurrent.ScheduledThreadPoolExecutor; 038 039import org.apache.hadoop.conf.Configuration; 040import org.apache.hadoop.hbase.HBaseClassTestRule; 041import org.apache.hadoop.hbase.HBaseConfiguration; 042import org.apache.hadoop.hbase.HBaseTestingUtility; 043import org.apache.hadoop.hbase.HConstants; 044import org.apache.hadoop.hbase.ServerName; 045import org.apache.hadoop.hbase.TableName; 046import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 047import org.apache.hadoop.hbase.client.Put; 048import org.apache.hadoop.hbase.client.RegionInfo; 049import org.apache.hadoop.hbase.client.Table; 050import org.apache.hadoop.hbase.testclassification.LargeTests; 051import org.apache.hadoop.hbase.util.Bytes; 052import org.apache.hadoop.util.ToolRunner; 053import org.apache.log4j.Appender; 054import org.apache.log4j.LogManager; 055import org.apache.log4j.spi.LoggingEvent; 056import org.junit.After; 057import org.junit.Before; 058import org.junit.ClassRule; 059import org.junit.Rule; 060import org.junit.Test; 061import org.junit.experimental.categories.Category; 062import org.junit.rules.TestName; 063import org.junit.runner.RunWith; 064import org.mockito.ArgumentMatcher; 065import org.mockito.Mock; 066import org.mockito.junit.MockitoJUnitRunner; 067import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; 068 069@RunWith(MockitoJUnitRunner.class) 070@Category({LargeTests.class}) 071public class TestCanaryTool { 072 073 @ClassRule 074 public static final HBaseClassTestRule CLASS_RULE = 075 HBaseClassTestRule.forClass(TestCanaryTool.class); 076 077 private HBaseTestingUtility testingUtility; 078 private static final byte[] FAMILY = Bytes.toBytes("f"); 079 private static final byte[] COLUMN = Bytes.toBytes("col"); 080 081 @Rule 082 public TestName name = new TestName(); 083 084 @Before 085 public void setUp() throws Exception { 086 testingUtility = new HBaseTestingUtility(); 087 testingUtility.startMiniCluster(); 088 LogManager.getRootLogger().addAppender(mockAppender); 089 } 090 091 @After 092 public void tearDown() throws Exception { 093 testingUtility.shutdownMiniCluster(); 094 LogManager.getRootLogger().removeAppender(mockAppender); 095 } 096 097 @Mock 098 Appender mockAppender; 099 100 @Test 101 public void testBasicZookeeperCanaryWorks() throws Exception { 102 final String[] args = { "-t", "10000", "-zookeeper" }; 103 testZookeeperCanaryWithArgs(args); 104 } 105 106 @Test 107 public void testZookeeperCanaryPermittedFailuresArgumentWorks() throws Exception { 108 final String[] args = { "-t", "10000", "-zookeeper", "-treatFailureAsError", "-permittedZookeeperFailures", "1" }; 109 testZookeeperCanaryWithArgs(args); 110 } 111 112 @Test 113 public void testBasicCanaryWorks() throws Exception { 114 final TableName tableName = TableName.valueOf(name.getMethodName()); 115 Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY }); 116 // insert some test rows 117 for (int i=0; i<1000; i++) { 118 byte[] iBytes = Bytes.toBytes(i); 119 Put p = new Put(iBytes); 120 p.addColumn(FAMILY, COLUMN, iBytes); 121 table.put(p); 122 } 123 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 124 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 125 CanaryTool canary = new CanaryTool(executor, sink); 126 String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() }; 127 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 128 assertEquals("verify no read error count", 0, canary.getReadFailures().size()); 129 assertEquals("verify no write error count", 0, canary.getWriteFailures().size()); 130 verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class), 131 isA(ColumnFamilyDescriptor.class), anyLong()); 132 } 133 134 @Test 135 public void testCanaryRegionTaskReadAllCF() throws Exception { 136 final TableName tableName = TableName.valueOf(name.getMethodName()); 137 Table table = testingUtility.createTable(tableName, 138 new byte[][] { Bytes.toBytes("f1"), Bytes.toBytes("f2") }); 139 // insert some test rows 140 for (int i = 0; i < 1000; i++) { 141 byte[] iBytes = Bytes.toBytes(i); 142 Put p = new Put(iBytes); 143 p.addColumn(Bytes.toBytes("f1"), COLUMN, iBytes); 144 p.addColumn(Bytes.toBytes("f2"), COLUMN, iBytes); 145 table.put(p); 146 } 147 Configuration configuration = HBaseConfiguration.create(testingUtility.getConfiguration()); 148 String[] args = { "-t", "10000", "testCanaryRegionTaskReadAllCF" }; 149 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 150 for (boolean readAllCF : new boolean[] { true, false }) { 151 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 152 CanaryTool canary = new CanaryTool(executor, sink); 153 configuration.setBoolean(HConstants.HBASE_CANARY_READ_ALL_CF, readAllCF); 154 assertEquals(0, ToolRunner.run(configuration, canary, args)); 155 // the test table has two column family. If readAllCF set true, 156 // we expect read count is double of region count 157 int expectedReadCount = 158 readAllCF ? 2 * sink.getTotalExpectedRegions() : sink.getTotalExpectedRegions(); 159 assertEquals("canary region success count should equal total expected read count", 160 expectedReadCount, sink.getReadSuccessCount()); 161 Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap(); 162 assertFalse("verify region map has size > 0", regionMap.isEmpty()); 163 164 for (String regionName : regionMap.keySet()) { 165 for (CanaryTool.RegionTaskResult res : regionMap.get(regionName)) { 166 assertNotNull("verify getRegionNameAsString()", regionName); 167 assertNotNull("verify getRegionInfo()", res.getRegionInfo()); 168 assertNotNull("verify getTableName()", res.getTableName()); 169 assertNotNull("verify getTableNameAsString()", res.getTableNameAsString()); 170 assertNotNull("verify getServerName()", res.getServerName()); 171 assertNotNull("verify getServerNameAsString()", res.getServerNameAsString()); 172 assertNotNull("verify getColumnFamily()", res.getColumnFamily()); 173 assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString()); 174 assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess()); 175 assertTrue("read took some time", res.getReadLatency() > -1); 176 } 177 } 178 } 179 } 180 181 @Test 182 public void testCanaryRegionTaskResult() throws Exception { 183 TableName tableName = TableName.valueOf("testCanaryRegionTaskResult"); 184 Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY }); 185 // insert some test rows 186 for (int i=0; i<1000; i++) { 187 byte[] iBytes = Bytes.toBytes(i); 188 Put p = new Put(iBytes); 189 p.addColumn(FAMILY, COLUMN, iBytes); 190 table.put(p); 191 } 192 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 193 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 194 CanaryTool canary = new CanaryTool(executor, sink); 195 String[] args = { "-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult" }; 196 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 197 198 assertTrue("canary should expect to scan at least 1 region", 199 sink.getTotalExpectedRegions() > 0); 200 assertTrue("there should be no read failures", sink.getReadFailureCount() == 0); 201 assertTrue("there should be no write failures", sink.getWriteFailureCount() == 0); 202 assertTrue("verify read success count > 0", sink.getReadSuccessCount() > 0); 203 assertTrue("verify write success count > 0", sink.getWriteSuccessCount() > 0); 204 verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class), 205 isA(ColumnFamilyDescriptor.class), anyLong()); 206 verify(sink, atLeastOnce()).publishWriteTiming(isA(ServerName.class), isA(RegionInfo.class), 207 isA(ColumnFamilyDescriptor.class), anyLong()); 208 209 assertEquals("canary region success count should equal total expected regions", 210 sink.getReadSuccessCount() + sink.getWriteSuccessCount(), sink.getTotalExpectedRegions()); 211 Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap(); 212 assertFalse("verify region map has size > 0", regionMap.isEmpty()); 213 214 for (String regionName : regionMap.keySet()) { 215 for (CanaryTool.RegionTaskResult res: regionMap.get(regionName)) { 216 assertNotNull("verify getRegionNameAsString()", regionName); 217 assertNotNull("verify getRegionInfo()", res.getRegionInfo()); 218 assertNotNull("verify getTableName()", res.getTableName()); 219 assertNotNull("verify getTableNameAsString()", res.getTableNameAsString()); 220 assertNotNull("verify getServerName()", res.getServerName()); 221 assertNotNull("verify getServerNameAsString()", res.getServerNameAsString()); 222 assertNotNull("verify getColumnFamily()", res.getColumnFamily()); 223 assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString()); 224 225 if (regionName.contains(CanaryTool.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) { 226 assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess()); 227 assertTrue("write took some time", res.getWriteLatency() > -1); 228 } else { 229 assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess()); 230 assertTrue("read took some time", res.getReadLatency() > -1); 231 } 232 } 233 } 234 } 235 236 // Ignore this test. It fails w/ the below on some mac os x. 237 // [ERROR] Failures: 238 // [ERROR] TestCanaryTool.testReadTableTimeouts:216 239 // Argument(s) are different! Wanted: 240 // mockAppender.doAppend( 241 // <custom argument matcher> 242 // ); 243 // -> at org.apache.hadoop.hbase.tool.TestCanaryTool 244 // .testReadTableTimeouts(TestCanaryTool.java:216) 245 // Actual invocations have different arguments: 246 // mockAppender.doAppend( 247 // org.apache.log4j.spi.LoggingEvent@2055cfc1 248 // ); 249 // ) 250 // ) 251 // 252 @org.junit.Ignore @Test 253 public void testReadTableTimeouts() throws Exception { 254 final TableName [] tableNames = new TableName[] {TableName.valueOf(name.getMethodName() + "1"), 255 TableName.valueOf(name.getMethodName() + "2")}; 256 // Create 2 test tables. 257 for (int j = 0; j < 2; j++) { 258 Table table = testingUtility.createTable(tableNames[j], new byte[][] { FAMILY }); 259 // insert some test rows 260 for (int i = 0; i < 10; i++) { 261 byte[] iBytes = Bytes.toBytes(i + j); 262 Put p = new Put(iBytes); 263 p.addColumn(FAMILY, COLUMN, iBytes); 264 table.put(p); 265 } 266 } 267 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 268 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 269 CanaryTool canary = new CanaryTool(executor, sink); 270 String configuredTimeoutStr = tableNames[0].getNameAsString() + "=" + Long.MAX_VALUE + "," + 271 tableNames[1].getNameAsString() + "=0"; 272 String[] args = {"-readTableTimeouts", configuredTimeoutStr, name.getMethodName() + "1", 273 name.getMethodName() + "2"}; 274 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 275 verify(sink, times(tableNames.length)).initializeAndGetReadLatencyForTable(isA(String.class)); 276 for (int i = 0; i < 2; i++) { 277 assertNotEquals("verify non-null read latency", null, 278 sink.getReadLatencyMap().get(tableNames[i].getNameAsString())); 279 assertNotEquals("verify non-zero read latency", 0L, 280 sink.getReadLatencyMap().get(tableNames[i].getNameAsString())); 281 } 282 // One table's timeout is set for 0 ms and thus, should lead to an error. 283 verify(mockAppender, times(1)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() { 284 @Override 285 public boolean matches(LoggingEvent argument) { 286 return argument.getRenderedMessage().contains("exceeded the configured read timeout."); 287 } 288 })); 289 verify(mockAppender, times(2)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() { 290 @Override 291 public boolean matches(LoggingEvent argument) { 292 return argument.getRenderedMessage().contains("Configured read timeout"); 293 } 294 })); 295 } 296 297 @Test 298 public void testWriteTableTimeout() throws Exception { 299 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 300 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 301 CanaryTool canary = new CanaryTool(executor, sink); 302 String[] args = { "-writeSniffing", "-writeTableTimeout", String.valueOf(Long.MAX_VALUE)}; 303 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 304 assertNotEquals("verify non-null write latency", null, sink.getWriteLatency()); 305 assertNotEquals("verify non-zero write latency", 0L, sink.getWriteLatency()); 306 verify(mockAppender, times(1)).doAppend(argThat( 307 new ArgumentMatcher<LoggingEvent>() { 308 @Override 309 public boolean matches(LoggingEvent argument) { 310 return argument.getRenderedMessage().contains("Configured write timeout"); 311 } 312 })); 313 } 314 315 //no table created, so there should be no regions 316 @Test 317 public void testRegionserverNoRegions() throws Exception { 318 runRegionserverCanary(); 319 verify(mockAppender).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() { 320 @Override 321 public boolean matches(LoggingEvent argument) { 322 return argument.getRenderedMessage().contains("Regionserver not serving any regions"); 323 } 324 })); 325 } 326 327 //by creating a table, there shouldn't be any region servers not serving any regions 328 @Test 329 public void testRegionserverWithRegions() throws Exception { 330 final TableName tableName = TableName.valueOf(name.getMethodName()); 331 testingUtility.createTable(tableName, new byte[][] { FAMILY }); 332 runRegionserverCanary(); 333 verify(mockAppender, never()).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() { 334 @Override 335 public boolean matches(LoggingEvent argument) { 336 return argument.getRenderedMessage().contains("Regionserver not serving any regions"); 337 } 338 })); 339 } 340 341 @Test 342 public void testRawScanConfig() throws Exception { 343 final TableName tableName = TableName.valueOf(name.getMethodName()); 344 Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY }); 345 // insert some test rows 346 for (int i=0; i<1000; i++) { 347 byte[] iBytes = Bytes.toBytes(i); 348 Put p = new Put(iBytes); 349 p.addColumn(FAMILY, COLUMN, iBytes); 350 table.put(p); 351 } 352 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 353 CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink()); 354 CanaryTool canary = new CanaryTool(executor, sink); 355 String[] args = { "-t", "10000", name.getMethodName() }; 356 org.apache.hadoop.conf.Configuration conf = 357 new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration()); 358 conf.setBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, true); 359 assertEquals(0, ToolRunner.run(conf, canary, args)); 360 verify(sink, atLeastOnce()) 361 .publishReadTiming(isA(ServerName.class), isA(RegionInfo.class), 362 isA(ColumnFamilyDescriptor.class), anyLong()); 363 assertEquals("verify no read error count", 0, canary.getReadFailures().size()); 364 } 365 366 private void runRegionserverCanary() throws Exception { 367 ExecutorService executor = new ScheduledThreadPoolExecutor(1); 368 CanaryTool canary = new CanaryTool(executor, new CanaryTool.RegionServerStdOutSink()); 369 String[] args = { "-t", "10000", "-regionserver"}; 370 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 371 assertEquals("verify no read error count", 0, canary.getReadFailures().size()); 372 } 373 374 private void testZookeeperCanaryWithArgs(String[] args) throws Exception { 375 Integer port = 376 Iterables.getOnlyElement(testingUtility.getZkCluster().getClientPortList(), null); 377 String hostPort = testingUtility.getZkCluster().getAddress().toString(); 378 testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM, hostPort + "/hbase"); 379 ExecutorService executor = new ScheduledThreadPoolExecutor(2); 380 CanaryTool.ZookeeperStdOutSink sink = spy(new CanaryTool.ZookeeperStdOutSink()); 381 CanaryTool canary = new CanaryTool(executor, sink); 382 assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); 383 384 String baseZnode = testingUtility.getConfiguration() 385 .get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); 386 verify(sink, atLeastOnce()).publishReadTiming(eq(baseZnode), eq(hostPort), anyLong()); 387 } 388}