001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.http; 019 020import static org.apache.hadoop.hbase.client.RegionInfoBuilder.FIRST_META_REGIONINFO; 021import static org.hamcrest.MatcherAssert.assertThat; 022import static org.hamcrest.Matchers.allOf; 023import static org.hamcrest.Matchers.containsString; 024import static org.hamcrest.Matchers.endsWith; 025import static org.hamcrest.Matchers.startsWith; 026import static org.junit.jupiter.api.Assertions.assertThrows; 027import static org.mockito.Mockito.mock; 028import static org.mockito.Mockito.when; 029 030import java.time.Instant; 031import java.util.ArrayList; 032import java.util.Arrays; 033import java.util.HashMap; 034import java.util.List; 035import java.util.Map; 036import java.util.Objects; 037import java.util.concurrent.CompletableFuture; 038import java.util.function.Supplier; 039import org.apache.hadoop.conf.Configuration; 040import org.apache.hadoop.fs.Path; 041import org.apache.hadoop.hbase.ConnectionExtension; 042import org.apache.hadoop.hbase.HBaseConfiguration; 043import org.apache.hadoop.hbase.HConstants; 044import org.apache.hadoop.hbase.MiniClusterExtension; 045import org.apache.hadoop.hbase.ServerName; 046import org.apache.hadoop.hbase.StartTestingClusterOption; 047import org.apache.hadoop.hbase.TableName; 048import org.apache.hadoop.hbase.client.AsyncAdmin; 049import org.apache.hadoop.hbase.client.AsyncConnection; 050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 051import org.apache.hadoop.hbase.client.Durability; 052import org.apache.hadoop.hbase.client.RegionInfo; 053import org.apache.hadoop.hbase.client.TableDescriptor; 054import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 055import org.apache.hadoop.hbase.master.HMaster; 056import org.apache.hadoop.hbase.master.hbck.HbckChore; 057import org.apache.hadoop.hbase.master.hbck.HbckReport; 058import org.apache.hadoop.hbase.master.http.hbck.resource.HbckMetricsResource; 059import org.apache.hadoop.hbase.master.janitor.CatalogJanitor; 060import org.apache.hadoop.hbase.master.janitor.CatalogJanitorReport; 061import org.apache.hadoop.hbase.testclassification.LargeTests; 062import org.apache.hadoop.hbase.testclassification.MasterTests; 063import org.apache.hadoop.hbase.util.Bytes; 064import org.apache.hadoop.hbase.util.Pair; 065import org.junit.jupiter.api.Order; 066import org.junit.jupiter.api.Tag; 067import org.junit.jupiter.api.Test; 068import org.junit.jupiter.api.extension.AfterAllCallback; 069import org.junit.jupiter.api.extension.BeforeAllCallback; 070import org.junit.jupiter.api.extension.ExtensionContext; 071import org.junit.jupiter.api.extension.RegisterExtension; 072import org.mockito.Mockito; 073import org.slf4j.Logger; 074import org.slf4j.LoggerFactory; 075 076import org.apache.hbase.thirdparty.javax.ws.rs.NotAcceptableException; 077import org.apache.hbase.thirdparty.javax.ws.rs.client.Client; 078import org.apache.hbase.thirdparty.javax.ws.rs.client.ClientBuilder; 079import org.apache.hbase.thirdparty.javax.ws.rs.client.WebTarget; 080import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; 081 082/** 083 * Tests for the {@link HbckMetricsResource}. 084 */ 085@Tag(MasterTests.TAG) 086@Tag(LargeTests.TAG) 087public class TestHbckMetricsResource { 088 089 private static final Logger LOG = LoggerFactory.getLogger(TestHbckMetricsResource.class); 090 091 // Test data for Mock HBCK Report 092 private static final long reportStartTime = 123456789000L; 093 private static final long reportEndTime = 234567890000L; 094 private static final String regionId1 = "regionId1"; 095 private static final String regionId2 = "regionId2"; 096 private static final String localhost1 = "localhost1"; 097 private static final String localhost2 = "localhost2"; 098 private static final String port = "16010"; 099 private static final String hostStartCode = "123456789"; 100 private static final String path1 = "hdfs://path1"; 101 private static final String path2 = "hdfs://path2"; 102 private static final String metaRegionID = FIRST_META_REGIONINFO.getEncodedName(); 103 private static final String metaTableName = FIRST_META_REGIONINFO.getTable().getNameAsString(); 104 105 // Various Keys in HBCK JSON Response. 106 private static final String quoteColon = "\":"; 107 private static final String quote = "\""; 108 private static final String regionId = quote + "region_id" + quoteColon; 109 private static final String regionHdfsPath = quote + "region_hdfs_path" + quoteColon; 110 private static final String rsName = quote + "rs_name" + quoteColon; 111 private static final String hostName = quote + "host_name" + quoteColon; 112 private static final String hostPort = quote + "host_port" + quoteColon; 113 private static final String startCode = quote + "start_code" + quoteColon; 114 private static final String serverNameInMeta = quote + "server_name_in_meta" + quoteColon; 115 private static final String listOfServers = quote + "list_of_servers" + quoteColon; 116 private static final String region1Info = quote + "region1_info" + quoteColon; 117 private static final String region2Info = quote + "region2_info" + quoteColon; 118 private static final String regionInfo = quote + "region_info" + quoteColon; 119 private static final String serverName = quote + "server_name" + quoteColon; 120 private static final String tableName = quote + "table_name" + quoteColon; 121 122 private static final String dataStartsWith = "{\"data\":["; 123 private static final String dataEndsWith = "]}"; 124 private static final String hbckReportStartTime = quote + "hbck_report_start_time" + quoteColon; 125 private static final String hbckReportEndTime = quote + "hbck_report_end_time" + quoteColon; 126 private static final String hbckOrphanRegionOnFS = 127 quote + "hbck_orphan_regions_on_fs" + quoteColon; 128 private static final String hbckOrphanRegionOnRS = 129 quote + "hbck_orphan_regions_on_rs" + quoteColon; 130 private static final String hbckInconsistentRegion = 131 quote + "hbck_inconsistent_regions" + quoteColon; 132 private static final String hbckHoles = quote + "hbck_holes" + quoteColon; 133 private static final String hbckOverlaps = quote + "hbck_overlaps" + quoteColon; 134 private static final String hbckUnknownServers = quote + "hbck_unknown_servers" + quoteColon; 135 private static final String hbckEmptyRegionInfo = quote + "hbck_empty_region_info" + quoteColon; 136 137 @Order(1) 138 @RegisterExtension 139 private static final MiniClusterExtension miniClusterExtension = MiniClusterExtension.newBuilder() 140 .setMiniClusterOption( 141 StartTestingClusterOption.builder().numZkServers(3).numMasters(3).numDataNodes(3).build()) 142 .setConfiguration(() -> { 143 // enable Master InfoServer and random port selection 144 final Configuration conf = HBaseConfiguration.create(); 145 conf.setInt(HConstants.MASTER_INFO_PORT, 0); 146 conf.set("hbase.http.jersey.tracing.type", "ON_DEMAND"); 147 return conf; 148 }).build(); 149 150 @Order(2) 151 @RegisterExtension 152 private static final ConnectionExtension connectionExtension = 153 ConnectionExtension.createAsyncConnectionExtension(miniClusterExtension::createAsyncConnection); 154 155 @Order(3) 156 @RegisterExtension 157 private static final ClassSetupExtension classRule = 158 new ClassSetupExtension(connectionExtension::getAsyncConnection); 159 160 private static final class ClassSetupExtension implements BeforeAllCallback, AfterAllCallback { 161 162 private final Supplier<AsyncConnection> connectionSupplier; 163 private final TableName tableName; 164 private AsyncAdmin admin; 165 private WebTarget target; 166 167 public ClassSetupExtension(final Supplier<AsyncConnection> connectionSupplier) { 168 this.connectionSupplier = connectionSupplier; 169 tableName = TableName.valueOf(TestHbckMetricsResource.class.getSimpleName()); 170 } 171 172 public WebTarget getTarget() { 173 return target; 174 } 175 176 @Override 177 public void beforeAll(ExtensionContext context) throws Exception { 178 final AsyncConnection conn = connectionSupplier.get(); 179 admin = conn.getAdmin(); 180 final TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 181 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("c")).build()) 182 .setDurability(Durability.SKIP_WAL).build(); 183 admin.createTable(tableDescriptor).get(); 184 185 HMaster master = miniClusterExtension.getTestingUtility().getMiniHBaseCluster().getMaster(); 186 187 HbckChore hbckChore = mock(HbckChore.class); 188 HbckReport hbckReport = mock(HbckReport.class); 189 CatalogJanitor catalogJanitorChore = mock(CatalogJanitor.class); 190 CatalogJanitorReport catalogJanitorReport = mock(CatalogJanitorReport.class); 191 master.setHbckChoreForTesting(hbckChore); 192 master.setCatalogJanitorChoreForTesting(catalogJanitorChore); 193 194 // Test data for Mock HBCK Report 195 ServerName server1 = 196 ServerName.valueOf(localhost1, Integer.parseInt(port), Integer.parseInt(hostStartCode)); 197 ServerName server2 = 198 ServerName.valueOf(localhost2, Integer.parseInt(port), Integer.parseInt(hostStartCode)); 199 Path hdfsPath1 = new Path(path1); 200 Path hdfsPath2 = new Path(path2); 201 202 // Orphan on RS Test data 203 Map<String, ServerName> mapOfOrphanRegionsOnRS = new HashMap<>(); 204 mapOfOrphanRegionsOnRS.put(regionId1, server1); 205 mapOfOrphanRegionsOnRS.put(regionId2, server2); 206 207 // Orphan Region on FS Test Data 208 Map<String, Path> mapOfOrphanRegionOnFS = new HashMap<>(); 209 mapOfOrphanRegionOnFS.put(regionId1, hdfsPath1); 210 mapOfOrphanRegionOnFS.put(regionId2, hdfsPath2); 211 212 // Inconsistent Regions Test Data 213 Map<String, Pair<ServerName, List<ServerName>>> mapOfInconsistentRegions = new HashMap<>(); 214 mapOfInconsistentRegions.put(regionId1, new Pair<>(server1, Arrays.asList(server1, server2))); 215 mapOfInconsistentRegions.put(regionId2, new Pair<>(server2, Arrays.asList(server1, server2))); 216 217 // Region Overlap and Region Holes Test Data 218 List<Pair<RegionInfo, RegionInfo>> listOfRegion = new ArrayList<>(); 219 listOfRegion.add(new Pair<>(FIRST_META_REGIONINFO, FIRST_META_REGIONINFO)); 220 listOfRegion.add(new Pair<>(FIRST_META_REGIONINFO, FIRST_META_REGIONINFO)); 221 222 // Unknown RegionServer Test Data 223 List<Pair<RegionInfo, ServerName>> listOfUnknownServers = new ArrayList<>(); 224 listOfUnknownServers.add(new Pair<>(FIRST_META_REGIONINFO, server1)); 225 listOfUnknownServers.add(new Pair<>(FIRST_META_REGIONINFO, server2)); 226 227 // Empty Region Info Test Data 228 List<byte[]> listOfEmptyRegionInfo = new ArrayList<>(); 229 listOfEmptyRegionInfo.add(regionId1.getBytes()); 230 listOfEmptyRegionInfo.add(regionId2.getBytes()); 231 232 // Mock HBCK Report and CatalogJanitor Report 233 when(hbckReport.getCheckingStartTimestamp()) 234 .thenReturn(Instant.ofEpochMilli(reportStartTime)); 235 when(hbckReport.getCheckingEndTimestamp()).thenReturn(Instant.ofEpochSecond(reportEndTime)); 236 when(hbckReport.getOrphanRegionsOnFS()).thenReturn(mapOfOrphanRegionOnFS); 237 when(hbckReport.getOrphanRegionsOnRS()).thenReturn(mapOfOrphanRegionsOnRS); 238 when(hbckReport.getInconsistentRegions()).thenReturn(mapOfInconsistentRegions); 239 when(catalogJanitorReport.getHoles()).thenReturn(listOfRegion); 240 when(catalogJanitorReport.getOverlaps()).thenReturn(listOfRegion); 241 when(catalogJanitorReport.getUnknownServers()).thenReturn(listOfUnknownServers); 242 when(catalogJanitorReport.getEmptyRegionInfo()).thenReturn(listOfEmptyRegionInfo); 243 244 Mockito.doReturn(hbckReport).when(hbckChore).getLastReport(); 245 Mockito.doReturn(catalogJanitorReport).when(catalogJanitorChore).getLastReport(); 246 247 final String baseUrl = 248 admin.getMaster().thenApply(ServerName::getHostname).thenCombine(admin.getMasterInfoPort(), 249 (hostName, infoPort) -> "http://" + hostName + ":" + infoPort).get(); 250 final Client client = ClientBuilder.newClient(); 251 target = client.target(baseUrl).path("hbck/hbck-metrics"); 252 } 253 254 @Override 255 public void afterAll(ExtensionContext context) { 256 final TableName tableName = TableName.valueOf("test"); 257 try { 258 admin.tableExists(tableName).thenCompose(val -> { 259 if (val) { 260 return admin.disableTable(tableName) 261 .thenCompose(ignored -> admin.deleteTable(tableName)); 262 } else { 263 return CompletableFuture.completedFuture(null); 264 } 265 }).get(); 266 } catch (Exception e) { 267 throw new RuntimeException(e); 268 } 269 } 270 } 271 272 @Test 273 public void testGetRoot() { 274 final String response = classRule.getTarget().request(MediaType.APPLICATION_JSON_TYPE) 275 .header("X-Jersey-Tracing-Accept", true).get(String.class); 276 LOG.info("HBCK JSON Response : " + response); 277 assertThat(response, 278 allOf(containsString(hbckReportStartTime), containsString(hbckReportEndTime), 279 containsString(hbckOrphanRegionOnFS), containsString(hbckOrphanRegionOnRS), 280 containsString(hbckInconsistentRegion), containsString(hbckHoles), 281 containsString(hbckOverlaps), containsString(hbckUnknownServers), 282 containsString(hbckEmptyRegionInfo), containsString(Objects.toString(reportStartTime)), 283 containsString(Objects.toString(reportEndTime)))); 284 } 285 286 @Test 287 public void testGetRootHtml() { 288 assertThrows(NotAcceptableException.class, () -> classRule.getTarget() 289 .request(MediaType.TEXT_HTML_TYPE).header("X-Jersey-Tracing-Accept", true).get(String.class)); 290 } 291 292 @Test 293 public void testGetOrphanRegionOnFS() { 294 final String response = 295 classRule.getTarget().path("orphan-regions-on-fs").request(MediaType.APPLICATION_JSON_TYPE) 296 .header("X-Jersey-Tracing-Accept", true).get(String.class); 297 LOG.info("HBCK Response for resource orphan-regions-on-fs : " + response); 298 assertThat(response, 299 allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(regionId), 300 containsString(regionHdfsPath), containsString(regionId1), containsString(regionId2), 301 containsString(path1), containsString(path2))); 302 } 303 304 @Test 305 public void testGetOrphanRegionOnFSHtml() { 306 assertThrows(NotAcceptableException.class, 307 () -> classRule.getTarget().path("orphan-regions-on-fs").request(MediaType.TEXT_HTML_TYPE) 308 .header("X-Jersey-Tracing-Accept", true).get(String.class)); 309 } 310 311 @Test 312 public void testGetOrphanRegionOnRS() { 313 final String response = 314 classRule.getTarget().path("orphan-regions-on-rs").request(MediaType.APPLICATION_JSON_TYPE) 315 .header("X-Jersey-Tracing-Accept", true).get(String.class); 316 LOG.info("HBCK Response for resource orphan-regions-on-rs : " + response); 317 assertThat(response, 318 allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(regionId), 319 containsString(rsName), containsString(hostName), containsString(hostPort), 320 containsString(startCode), containsString(regionId1), containsString(regionId2), 321 containsString(localhost1), containsString(localhost2), containsString(port), 322 containsString(hostStartCode))); 323 } 324 325 @Test 326 public void testGetOrphanRegionOnRSHtml() { 327 assertThrows(NotAcceptableException.class, 328 () -> classRule.getTarget().path("orphan-regions-on-rs").request(MediaType.TEXT_HTML_TYPE) 329 .header("X-Jersey-Tracing-Accept", true).get(String.class)); 330 } 331 332 @Test 333 public void testGetInconsistentRegions() { 334 final String response = 335 classRule.getTarget().path("inconsistent-regions").request(MediaType.APPLICATION_JSON_TYPE) 336 .header("X-Jersey-Tracing-Accept", true).get(String.class); 337 LOG.info("HBCK Response for resource inconsistent-regions : " + response); 338 assertThat(response, 339 allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(hostName), 340 containsString(hostPort), containsString(startCode), containsString(listOfServers), 341 containsString(regionId1), containsString(regionId2), containsString(regionId), 342 containsString(serverNameInMeta), containsString(localhost1), containsString(localhost2), 343 containsString(port), containsString(hostStartCode))); 344 } 345 346 @Test 347 public void testGetInconsistentRegionsHtml() { 348 assertThrows(NotAcceptableException.class, 349 () -> classRule.getTarget().path("inconsistent-regions").request(MediaType.TEXT_HTML_TYPE) 350 .header("X-Jersey-Tracing-Accept", true).get(String.class)); 351 } 352 353 @Test 354 public void testGetRegionHoles() { 355 final String response = 356 classRule.getTarget().path("region-holes").request(MediaType.APPLICATION_JSON_TYPE) 357 .header("X-Jersey-Tracing-Accept", true).get(String.class); 358 LOG.info("HBCK Response for resource region-holes : " + response); 359 assertThat(response, 360 allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(region1Info), 361 containsString(region2Info), containsString(regionId), containsString(tableName), 362 containsString(metaRegionID), containsString(metaTableName))); 363 } 364 365 @Test 366 public void testGetRegionHolesHtml() { 367 assertThrows(NotAcceptableException.class, () -> classRule.getTarget().path("region-holes") 368 .request(MediaType.TEXT_HTML_TYPE).header("X-Jersey-Tracing-Accept", true).get(String.class)); 369 } 370 371 @Test 372 public void testGetRegionOverlaps() { 373 final String response = 374 classRule.getTarget().path("region-overlaps").request(MediaType.APPLICATION_JSON_TYPE) 375 .header("X-Jersey-Tracing-Accept", true).get(String.class); 376 LOG.info("HBCK Response for resource region-overlaps : " + response); 377 assertThat(response, 378 allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(regionId), 379 containsString(tableName), containsString(region2Info), containsString(region2Info), 380 containsString(metaRegionID), containsString(metaTableName))); 381 } 382 383 @Test 384 public void testGetRegionOverlapsHtml() { 385 assertThrows(NotAcceptableException.class, () -> classRule.getTarget().path("region-overlaps") 386 .request(MediaType.TEXT_HTML_TYPE).header("X-Jersey-Tracing-Accept", true).get(String.class)); 387 } 388 389 @Test 390 public void testGetUnkownServers() { 391 final String response = 392 classRule.getTarget().path("unknown-servers").request(MediaType.APPLICATION_JSON_TYPE) 393 .header("X-Jersey-Tracing-Accept", true).get(String.class); 394 LOG.info("HBCK Response for resource unknown-servers : " + response); 395 assertThat(response, 396 allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), containsString(regionInfo), 397 containsString(regionId), containsString(tableName), containsString(serverName), 398 containsString(serverName), containsString(port), containsString(startCode), 399 containsString(metaRegionID), containsString(metaTableName), containsString(localhost1), 400 containsString(localhost2), containsString(port), containsString(hostStartCode))); 401 } 402 403 @Test 404 public void testGetUnknownServersHtml() { 405 assertThrows(NotAcceptableException.class, () -> classRule.getTarget().path("unknown-servers") 406 .request(MediaType.TEXT_HTML_TYPE).header("X-Jersey-Tracing-Accept", true).get(String.class)); 407 } 408 409 @Test 410 public void testGetEmptyRegionInfo() { 411 final String response = 412 classRule.getTarget().path("empty-regioninfo").request(MediaType.APPLICATION_JSON_TYPE) 413 .header("X-Jersey-Tracing-Accept", true).get(String.class); 414 LOG.info("HBCK Response for resource empty-regioninfo : " + response); 415 assertThat(response, allOf(startsWith(dataStartsWith), endsWith(dataEndsWith), 416 containsString(regionInfo), containsString(regionId1), containsString(regionId2))); 417 } 418 419 @Test 420 public void testGetEmptyRegionInfoHtml() { 421 assertThrows(NotAcceptableException.class, () -> classRule.getTarget().path("empty-regioninfo") 422 .request(MediaType.TEXT_HTML_TYPE).header("X-Jersey-Tracing-Accept", true).get(String.class)); 423 } 424}