001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.io.IOException;
025import java.io.InterruptedIOException;
026import java.io.UncheckedIOException;
027import java.util.ArrayList;
028import java.util.EnumSet;
029import java.util.HashMap;
030import java.util.List;
031import java.util.Map;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.ClusterMetrics;
036import org.apache.hadoop.hbase.CompatibilityFactory;
037import org.apache.hadoop.hbase.HBaseTestingUtil;
038import org.apache.hadoop.hbase.NamespaceDescriptor;
039import org.apache.hadoop.hbase.ServerMetricsBuilder;
040import org.apache.hadoop.hbase.ServerName;
041import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
042import org.apache.hadoop.hbase.StartTestingClusterOption;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.YouAreDeadException;
045import org.apache.hadoop.hbase.client.Admin;
046import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
047import org.apache.hadoop.hbase.client.RegionStatesCount;
048import org.apache.hadoop.hbase.client.TableDescriptor;
049import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
050import org.apache.hadoop.hbase.test.MetricsAssertHelper;
051import org.apache.hadoop.hbase.testclassification.MasterTests;
052import org.apache.hadoop.hbase.testclassification.MediumTests;
053import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
054import org.apache.hadoop.hbase.util.FSTableDescriptors;
055import org.apache.hadoop.hbase.util.IOExceptionConsumer;
056import org.apache.zookeeper.KeeperException;
057import org.junit.jupiter.api.AfterAll;
058import org.junit.jupiter.api.BeforeAll;
059import org.junit.jupiter.api.Tag;
060import org.junit.jupiter.api.Test;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063
064import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
065import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
066
067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
068import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos;
069import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos;
070import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
071import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
072
073@Tag(MasterTests.TAG)
074@Tag(MediumTests.TAG)
075public class TestMasterMetrics {
076
077  private static final Logger LOG = LoggerFactory.getLogger(TestMasterMetrics.class);
078  private static final MetricsAssertHelper metricsHelper =
079    CompatibilityFactory.getInstance(MetricsAssertHelper.class);
080  private static final String COLUMN_FAMILY = "cf";
081
082  private static SingleProcessHBaseCluster cluster;
083  private static HMaster master;
084  private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
085
086  public static class MyMaster extends HMaster {
087
088    public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
089      super(conf);
090    }
091
092    @Override
093    protected MasterRpcServices createRpcServices() throws IOException {
094      return new MasterRpcServices(this) {
095
096        @Override
097        public RegionServerStartupResponse regionServerStartup(RpcController controller,
098          RegionServerStartupRequest request) throws ServiceException {
099          RegionServerStartupResponse resp = super.regionServerStartup(controller, request);
100          ServerManager serverManager = getServerManager();
101          // to let the region server actual online otherwise we can not assign meta region
102          new HashMap<>(serverManager.getOnlineServers()).forEach((sn, sm) -> {
103            if (sm.getLastReportTimestamp() <= 0) {
104              try {
105                serverManager.regionServerReport(sn,
106                  ServerMetricsBuilder.newBuilder(sn).setVersionNumber(sm.getVersionNumber())
107                    .setVersion(sm.getVersion())
108                    .setLastReportTimestamp(EnvironmentEdgeManager.currentTime()).build());
109              } catch (YouAreDeadException e) {
110                throw new UncheckedIOException(e);
111              }
112            }
113          });
114          return resp;
115        }
116      };
117    }
118  }
119
120  public static class MyRegionServer
121    extends SingleProcessHBaseCluster.MiniHBaseClusterRegionServer {
122
123    public MyRegionServer(Configuration conf) throws IOException, InterruptedException {
124      super(conf);
125    }
126
127    @Override
128    protected void tryRegionServerReport(long reportStartTime, long reportEndTime) {
129      // do nothing
130    }
131  }
132
133  @BeforeAll
134  public static void startCluster() throws Exception {
135    LOG.info("Starting cluster");
136    // Set master class and use default values for other options.
137    StartTestingClusterOption option = StartTestingClusterOption.builder()
138      .masterClass(MyMaster.class).rsClass(MyRegionServer.class).build();
139    TEST_UTIL.startMiniCluster(option);
140    cluster = TEST_UTIL.getHBaseCluster();
141    LOG.info("Waiting for active/ready master");
142    cluster.waitForActiveAndReadyMaster();
143    master = cluster.getMaster();
144  }
145
146  @AfterAll
147  public static void after() throws Exception {
148    master.stopMaster();
149    TEST_UTIL.shutdownMiniCluster();
150  }
151
152  @Test
153  public void testSwitchMetricsFollowAdminChanges() throws IOException {
154    Admin admin = TEST_UTIL.getAdmin();
155    MetricsMasterSource source = master.getMasterMetrics().getMetricsSource();
156    Map<String, IOExceptionConsumer<Boolean>> switches = new HashMap<>();
157    switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME,
158      enabled -> admin.balancerSwitch(enabled, false));
159    switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, admin::normalizerSwitch);
160    switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME,
161      enabled -> admin.splitSwitch(enabled, false));
162    switches.put(MetricsMasterSource.MERGE_ENABLED_NAME,
163      enabled -> admin.mergeSwitch(enabled, false));
164    switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, admin::cleanerChoreSwitch);
165    switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, admin::catalogJanitorSwitch);
166    switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME,
167      enabled -> admin.snapshotCleanupSwitch(enabled, false));
168    switches.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME,
169      admin::replicationPeerModificationSwitch);
170    Map<String, Integer> initialStates = new MetricsMasterWrapperImpl(master).getSwitchStates();
171    for (Map.Entry<String, IOExceptionConsumer<Boolean>> entry : switches.entrySet()) {
172      try {
173        for (boolean enabled : new boolean[] { false, true }) {
174          entry.getValue().accept(enabled);
175          metricsHelper.assertGauge(entry.getKey(), enabled ? 1L : 0L, source);
176        }
177      } finally {
178        entry.getValue().accept(initialStates.get(entry.getKey()) == 1);
179      }
180    }
181    // Quota support is disabled by default.
182    metricsHelper.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0L, source);
183  }
184
185  @Test
186  public void testClusterRequests() throws Exception {
187    // sending fake request to master to see how metric value has changed
188    RegionServerStatusProtos.RegionServerReportRequest.Builder request =
189      RegionServerStatusProtos.RegionServerReportRequest.newBuilder();
190    ServerName serverName = cluster.getMaster(0).getServerName();
191    request.setServer(ProtobufUtil.toServerName(serverName));
192    long expectedRequestNumber = 10000;
193
194    MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource();
195    ClusterStatusProtos.ServerLoad sl = ClusterStatusProtos.ServerLoad.newBuilder()
196      .setTotalNumberOfRequests(expectedRequestNumber).setReadRequestsCount(expectedRequestNumber)
197      .setWriteRequestsCount(expectedRequestNumber).build();
198    request.setLoad(sl);
199
200    master.getMasterRpcServices().regionServerReport(null, request.build());
201    metricsHelper.assertCounter("cluster_requests", expectedRequestNumber, masterSource);
202    metricsHelper.assertCounter("cluster_read_requests", expectedRequestNumber, masterSource);
203    metricsHelper.assertCounter("cluster_write_requests", expectedRequestNumber, masterSource);
204
205    expectedRequestNumber = 15000;
206
207    sl = ClusterStatusProtos.ServerLoad.newBuilder().setTotalNumberOfRequests(expectedRequestNumber)
208      .setReadRequestsCount(expectedRequestNumber).setWriteRequestsCount(expectedRequestNumber)
209      .build();
210    request.setLoad(sl);
211
212    master.getMasterRpcServices().regionServerReport(null, request.build());
213    metricsHelper.assertCounter("cluster_requests", expectedRequestNumber, masterSource);
214    metricsHelper.assertCounter("cluster_read_requests", expectedRequestNumber, masterSource);
215    metricsHelper.assertCounter("cluster_write_requests", expectedRequestNumber, masterSource);
216  }
217
218  @Test
219  public void testDefaultMasterMetrics() throws Exception {
220    MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource();
221    metricsHelper.assertGauge("numRegionServers", 1, masterSource);
222    metricsHelper.assertGauge("averageLoad", 1, masterSource);
223    metricsHelper.assertGauge("numDeadRegionServers", 0, masterSource);
224    metricsHelper.assertGauge("numDrainingRegionServers", 0, masterSource);
225
226    metricsHelper.assertGauge("masterStartTime", master.getMasterStartTime(), masterSource);
227    metricsHelper.assertGauge("masterActiveTime", master.getMasterActiveTime(), masterSource);
228
229    metricsHelper.assertTag("isActiveMaster", "true", masterSource);
230    metricsHelper.assertTag("serverName", master.getServerName().toString(), masterSource);
231    metricsHelper.assertTag("clusterId", master.getClusterId(), masterSource);
232    metricsHelper.assertTag("zookeeperQuorum", master.getZooKeeper().getQuorum(), masterSource);
233
234    metricsHelper.assertCounter(MetricsMasterSource.SERVER_CRASH_METRIC_PREFIX + "SubmittedCount",
235      0, masterSource);
236    metricsHelper.assertGauge("oldWALsDirSize", master.getMasterWalManager().getOldWALsDirSize(),
237      masterSource);
238  }
239
240  @Test
241  public void testDefaultMasterProcMetrics() {
242    MetricsMasterProcSource masterSource = master.getMasterMetrics().getMetricsProcSource();
243    metricsHelper.assertGauge("numMasterWALs", master.getNumWALFiles(), masterSource);
244  }
245
246  // Verifies a foreign meta table does not show up in the table regions state
247  @Test
248  public void testClusterMetricsSkippingForeignMetaTable() throws Exception {
249    TableName replicaMetaTable = TableName.valueOf("hbase", "meta_replica");
250    TableDescriptor replicaMetaDescriptor = TableDescriptorBuilder.newBuilder(replicaMetaTable)
251      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("info")).build();
252    master.getTableDescriptors().update(replicaMetaDescriptor, true);
253    try {
254      Map<TableName, RegionStatesCount> tableRegionStatesCount = getTableRegionStatesCount();
255
256      assertFalse(tableRegionStatesCount.containsKey(replicaMetaTable),
257        "Foreign meta table should not be present");
258      assertTrue(tableRegionStatesCount.containsKey(TableName.META_TABLE_NAME),
259        "Local meta should be present");
260
261    } finally {
262      master.getTableDescriptors().remove(replicaMetaTable);
263    }
264  }
265
266  // This test adds foreign file descriptors to the cluster's table descriptor cache. It then
267  // verifies the foreign tables do not show up in the table regions state.
268  @Test
269  public void testClusterMetricsSkippingCachedForeignTables() throws Exception {
270    List<TableName> allTables = new ArrayList<>();
271
272    // These tables, including the cluster's meta table, should not be foreign to the cluster.
273    // The cluster should be able to find their state.
274    allTables.add(TableName.META_TABLE_NAME);
275    List<TableName> familiarTables = new ArrayList<>();
276    familiarTables.add(TableName.valueOf(null, "familiarTable1"));
277    familiarTables.add(TableName.valueOf("", "familiarTable2"));
278    familiarTables.add(TableName.valueOf("default", "familiarTable3"));
279    familiarTables.add(TableName.valueOf("familiarNamespace", "familiarTable4"));
280    familiarTables.add(TableName.valueOf("familiarNamespace", "familiarTable5"));
281
282    // Create these "familiar" tables so their state can be found
283    TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create("familiarNamespace").build());
284    for (TableName familiarTable : familiarTables) {
285      TEST_UTIL.createTable(familiarTable, COLUMN_FAMILY);
286      allTables.add(familiarTable);
287    }
288
289    // hbase:meta is a familiar table that was created automatically
290    familiarTables.add(TableName.META_TABLE_NAME);
291
292    // These tables should be foreign to the cluster.
293    // The cluster should not be able to find their state.
294    allTables.add(TableName.valueOf("hbase", "meta_replica"));
295    allTables.add(TableName.valueOf(null, "defaultNamespaceTable1"));
296    allTables.add(TableName.valueOf("", "defaultNamespaceTable2"));
297    allTables.add(TableName.valueOf("default", "defaultNamespaceTable3"));
298    allTables.add(TableName.valueOf("customNamespace", "customNamespaceTable1"));
299    allTables.add(TableName.valueOf("customNamespace", "customNamespaceTable2"));
300    allTables.add(TableName.valueOf("anotherNamespace", "anotherNamespaceTable"));
301    allTables.add(TableName.valueOf("sharedNamespace", "sharedNamespaceTable1"));
302    allTables.add(TableName.valueOf("sharedNamespace", "sharedNamespaceTable2"));
303
304    // Update master's table descriptors to have all tables
305    TableDescriptor foreignTableDescriptor;
306    for (TableName tableName : allTables) {
307      foreignTableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
308        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(COLUMN_FAMILY)).build();
309      master.getTableDescriptors().update(foreignTableDescriptor, true);
310    }
311
312    // The state of the meta table and the familiar tables we created should exist.
313    // The other tables' state should not exist.
314    for (TableName tableName : allTables) {
315      try {
316        Map<TableName, RegionStatesCount> tableRegionStatesCount = getTableRegionStatesCount();
317
318        if (
319          tableName.equals(TableName.META_TABLE_NAME)
320            || tableName.getQualifierAsString().startsWith("familiar")
321        ) {
322          assertTrue(tableRegionStatesCount.containsKey(tableName),
323            "Expected this table's state to exist: " + tableName);
324        } else {
325          assertFalse(tableRegionStatesCount.containsKey(tableName),
326            "This foreign table's state should not exist: " + tableName);
327        }
328      } finally {
329        if (!TableName.META_TABLE_NAME.equals(tableName) && familiarTables.contains(tableName)) {
330          LOG.debug("Deleting table: {}", tableName);
331          TEST_UTIL.deleteTable(tableName);
332        } else if (!familiarTables.contains(tableName)) {
333          LOG.debug("Removing table descriptor for foreign table: {}", tableName);
334          master.getTableDescriptors().remove(tableName);
335        }
336      }
337    }
338  }
339
340  // This test creates foreign file descriptors on the filesystem in addition to updating the
341  // table descriptor cache. It then verifies the foreign tables do not show up in the
342  // table regions state.
343  @Test
344  public void testClusterMetricsSkippingForeignTablesOnFileSystem() throws IOException {
345    List<TableName> familiarTables = new ArrayList<>();
346    List<TableName> foreignTables = new ArrayList<>();
347    FileSystem fs = TEST_UTIL.getTestFileSystem();
348    Path testDir = TEST_UTIL.getDataTestDirOnTestFS();
349    LOG.info("The test dir is: {}", testDir);
350
351    // Create tables whose state are familiar to this cluster
352    familiarTables.add(TableName.valueOf("testTable1"));
353    familiarTables.add(TableName.valueOf("testTable2"));
354    TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create("myNamespace").build());
355    familiarTables.add(TableName.valueOf("myNamespace", "tableWithNamespace1"));
356    for (TableName tableName : familiarTables) {
357      TEST_UTIL.createTable(tableName, COLUMN_FAMILY);
358    }
359
360    // hbase:meta is a familiar table that was created automatically
361    familiarTables.add(TableName.META_TABLE_NAME);
362
363    // There should now be 4 tables, including hbase:meta
364    Map<String, TableDescriptor> tableDescriptorMap = master.getTableDescriptors().getAll();
365    assertEquals(4, tableDescriptorMap.size());
366    for (TableName tableName : familiarTables) {
367      assertTrue(
368        tableDescriptorMap
369          .containsKey(tableName.getNamespaceAsString() + ":" + tableName.getQualifierAsString()),
370        "Expected table descriptor map to contain table: " + tableName);
371    }
372
373    createTableDescriptorOnFileSystem("hbase", "meta_replica", foreignTables);
374    createTableDescriptorOnFileSystem("default", "foreignTable1", foreignTables);
375    createTableDescriptorOnFileSystem("customForeignNs", "customForeignNsTable1", foreignTables);
376
377    // Verify the table descriptors were created on the filesystem
378    for (TableName tableName : foreignTables) {
379      Path tableDescPath = new Path(testDir,
380        "data" + Path.SEPARATOR + tableName.getNamespaceAsString() + Path.SEPARATOR
381          + tableName.getQualifierAsString() + Path.SEPARATOR + FSTableDescriptors.TABLEINFO_DIR);
382      assertTrue(fs.exists(tableDescPath),
383        "Expected table descriptor directory to exist: " + tableDescPath);
384    }
385
386    Map<TableName, RegionStatesCount> tableRegionStatesCount = getTableRegionStatesCount();
387
388    // The foreign tables should not be in the table state
389    assertEquals(4, tableRegionStatesCount.size());
390    for (TableName tableName : familiarTables) {
391      assertTrue(tableRegionStatesCount.containsKey(tableName),
392        "Expected table regions state count to contain: " + tableName);
393      // Delete unneeded tables
394      if (!TableName.META_TABLE_NAME.equals(tableName)) {
395        LOG.debug("Deleting table: {}", tableName);
396        TEST_UTIL.deleteTable(tableName);
397      }
398    }
399    for (TableName tableName : foreignTables) {
400      assertFalse(tableRegionStatesCount.containsKey(tableName),
401        "Expected table regions state count to NOT contain: " + tableName);
402      // Remove unneeded table descriptors
403      LOG.debug("Removing table descriptor for foreign table: {}", tableName);
404      master.getTableDescriptors().remove(tableName);
405    }
406  }
407
408  private Map<TableName, RegionStatesCount> getTableRegionStatesCount()
409    throws InterruptedIOException {
410    ClusterMetrics metrics = master.getClusterMetricsWithoutCoprocessor(
411      EnumSet.of(ClusterMetrics.Option.TABLE_TO_REGIONS_COUNT));
412    return metrics.getTableRegionStatesCount();
413  }
414
415  private void createTableDescriptorOnFileSystem(String namespace, String qualifier,
416    List<TableName> tableNameList) throws IOException {
417    // Create a table descriptor on the filesystem that uses the provided namespace
418    TableName tableName = TableName.valueOf(namespace, qualifier);
419    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
420      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(COLUMN_FAMILY)).build();
421    // Create the table descriptor on the filesystem and update the file descriptor cache
422    master.getTableDescriptors().update(tableDescriptor, false);
423
424    tableNameList.add(tableName);
425  }
426}