001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.coprocessor;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025
026import java.io.IOException;
027import java.util.List;
028import java.util.Optional;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.CoprocessorEnvironment;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtil;
034import org.apache.hadoop.hbase.HRegionLocation;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.Admin;
037import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
038import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
039import org.apache.hadoop.hbase.client.Connection;
040import org.apache.hadoop.hbase.client.ConnectionFactory;
041import org.apache.hadoop.hbase.client.Get;
042import org.apache.hadoop.hbase.client.Mutation;
043import org.apache.hadoop.hbase.client.Put;
044import org.apache.hadoop.hbase.client.RegionInfo;
045import org.apache.hadoop.hbase.client.RegionLocator;
046import org.apache.hadoop.hbase.client.Table;
047import org.apache.hadoop.hbase.client.TableDescriptor;
048import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
049import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
050import org.apache.hadoop.hbase.metrics.Counter;
051import org.apache.hadoop.hbase.metrics.Metric;
052import org.apache.hadoop.hbase.metrics.MetricRegistries;
053import org.apache.hadoop.hbase.metrics.MetricRegistry;
054import org.apache.hadoop.hbase.metrics.MetricRegistryInfo;
055import org.apache.hadoop.hbase.metrics.Timer;
056import org.apache.hadoop.hbase.regionserver.HRegionServer;
057import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
058import org.apache.hadoop.hbase.testclassification.MediumTests;
059import org.apache.hadoop.hbase.util.Bytes;
060import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
061import org.apache.hadoop.hbase.wal.WALEdit;
062import org.apache.hadoop.hbase.wal.WALKey;
063import org.junit.AfterClass;
064import org.junit.Before;
065import org.junit.BeforeClass;
066import org.junit.ClassRule;
067import org.junit.Rule;
068import org.junit.Test;
069import org.junit.experimental.categories.Category;
070import org.junit.rules.TestName;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
075import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
076import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
077import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
078
079import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
080import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
081import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
082import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
083import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MutateRowsResponse;
084
085/**
086 * Testing of coprocessor metrics end-to-end.
087 */
088@Category({ CoprocessorTests.class, MediumTests.class })
089public class TestCoprocessorMetrics {
090
091  @ClassRule
092  public static final HBaseClassTestRule CLASS_RULE =
093    HBaseClassTestRule.forClass(TestCoprocessorMetrics.class);
094
095  private static final Logger LOG = LoggerFactory.getLogger(TestCoprocessorMetrics.class);
096  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
097
098  private static final byte[] foo = Bytes.toBytes("foo");
099  private static final byte[] bar = Bytes.toBytes("bar");
100
101  @Rule
102  public TestName name = new TestName();
103
104  /**
105   * MasterObserver that has a Timer metric for create table operation.
106   */
107  public static class CustomMasterObserver implements MasterCoprocessor, MasterObserver {
108    private Timer createTableTimer;
109    private long start = Long.MIN_VALUE;
110
111    @Override
112    public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
113      TableDescriptor desc, RegionInfo[] regions) throws IOException {
114      // we rely on the fact that there is only 1 instance of our MasterObserver
115      this.start = EnvironmentEdgeManager.currentTime();
116    }
117
118    @Override
119    public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
120      TableDescriptor desc, RegionInfo[] regions) throws IOException {
121      if (this.start > 0) {
122        long time = EnvironmentEdgeManager.currentTime() - start;
123        LOG.info("Create table took: " + time);
124        createTableTimer.updateMillis(time);
125      }
126    }
127
128    @Override
129    public void start(CoprocessorEnvironment env) throws IOException {
130      if (env instanceof MasterCoprocessorEnvironment) {
131        MetricRegistry registry = ((MasterCoprocessorEnvironment) env).getMetricRegistryForMaster();
132
133        createTableTimer = registry.timer("CreateTable");
134      }
135    }
136
137    @Override
138    public Optional<MasterObserver> getMasterObserver() {
139      return Optional.of(this);
140    }
141  }
142
143  /**
144   * RegionServerObserver that has a Counter for rollWAL requests.
145   */
146  public static class CustomRegionServerObserver
147    implements RegionServerCoprocessor, RegionServerObserver {
148    /** This is the Counter metric object to keep track of the current count across invocations */
149    private Counter rollWALCounter;
150
151    @Override
152    public Optional<RegionServerObserver> getRegionServerObserver() {
153      return Optional.of(this);
154    }
155
156    @Override
157    public void postRollWALWriterRequest(ObserverContext<RegionServerCoprocessorEnvironment> ctx)
158      throws IOException {
159      // Increment the Counter whenever the coprocessor is called
160      rollWALCounter.increment();
161    }
162
163    @Override
164    public void start(CoprocessorEnvironment env) throws IOException {
165      if (env instanceof RegionServerCoprocessorEnvironment) {
166        MetricRegistry registry =
167          ((RegionServerCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
168
169        if (rollWALCounter == null) {
170          rollWALCounter = registry.counter("rollWALRequests");
171        }
172      }
173    }
174  }
175
176  /**
177   * WALObserver that has a Counter for walEdits written.
178   */
179  public static class CustomWALObserver implements WALCoprocessor, WALObserver {
180    private Counter walEditsCount;
181
182    @Override
183    public void postWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
184      RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
185      walEditsCount.increment();
186    }
187
188    @Override
189    public void start(CoprocessorEnvironment env) throws IOException {
190      if (env instanceof WALCoprocessorEnvironment) {
191        MetricRegistry registry =
192          ((WALCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
193
194        if (walEditsCount == null) {
195          walEditsCount = registry.counter("walEditsCount");
196        }
197      }
198    }
199
200    @Override
201    public Optional<WALObserver> getWALObserver() {
202      return Optional.of(this);
203    }
204  }
205
206  /**
207   * RegionObserver that has a Counter for preGet()
208   */
209  public static class CustomRegionObserver implements RegionCoprocessor, RegionObserver {
210    private Counter preGetCounter;
211
212    @Override
213    public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get,
214      List<Cell> results) throws IOException {
215      preGetCounter.increment();
216    }
217
218    @Override
219    public Optional<RegionObserver> getRegionObserver() {
220      return Optional.of(this);
221    }
222
223    @Override
224    public void start(CoprocessorEnvironment env) throws IOException {
225      if (env instanceof RegionCoprocessorEnvironment) {
226        MetricRegistry registry =
227          ((RegionCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
228
229        if (preGetCounter == null) {
230          preGetCounter = registry.counter("preGetRequests");
231        }
232      }
233    }
234  }
235
236  public static class CustomRegionObserver2 extends CustomRegionObserver {
237  }
238
239  /**
240   * RegionEndpoint to test metrics from endpoint calls
241   */
242  public static class CustomRegionEndpoint extends MultiRowMutationEndpoint {
243
244    private Timer endpointExecution;
245
246    @Override
247    public void mutateRows(RpcController controller, MutateRowsRequest request,
248      RpcCallback<MutateRowsResponse> done) {
249      long start = System.nanoTime();
250      super.mutateRows(controller, request, done);
251      endpointExecution.updateNanos(System.nanoTime() - start);
252    }
253
254    @Override
255    public void start(CoprocessorEnvironment env) throws IOException {
256      super.start(env);
257
258      if (env instanceof RegionCoprocessorEnvironment) {
259        MetricRegistry registry =
260          ((RegionCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
261
262        if (endpointExecution == null) {
263          endpointExecution = registry.timer("EndpointExecution");
264        }
265      }
266    }
267  }
268
269  @BeforeClass
270  public static void setupBeforeClass() throws Exception {
271    Configuration conf = UTIL.getConfiguration();
272    // inject master, regionserver and WAL coprocessors
273    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CustomMasterObserver.class.getName());
274    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
275      CustomRegionServerObserver.class.getName());
276    conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, CustomWALObserver.class.getName());
277    conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
278    UTIL.startMiniCluster();
279  }
280
281  @AfterClass
282  public static void teardownAfterClass() throws Exception {
283    UTIL.shutdownMiniCluster();
284  }
285
286  @Before
287  public void setup() throws IOException {
288    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
289      Admin admin = connection.getAdmin()) {
290      for (TableDescriptor htd : admin.listTableDescriptors()) {
291        UTIL.deleteTable(htd.getTableName());
292      }
293    }
294  }
295
296  @Test
297  public void testMasterObserver() throws IOException {
298    // Find out the MetricRegistry used by the CP using the global registries
299    MetricRegistryInfo info = MetricsCoprocessor
300      .createRegistryInfoForMasterCoprocessor(CustomMasterObserver.class.getName());
301    Optional<MetricRegistry> registry = MetricRegistries.global().get(info);
302    assertTrue(registry.isPresent());
303
304    Optional<Metric> metric = registry.get().get("CreateTable");
305    assertTrue(metric.isPresent());
306
307    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
308      Admin admin = connection.getAdmin()) {
309
310      Timer createTableTimer = (Timer) metric.get();
311      long prevCount = createTableTimer.getHistogram().getCount();
312      LOG.info("Creating table");
313      TableDescriptorBuilder tableDescriptorBuilder =
314        TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
315      ColumnFamilyDescriptor columnFamilyDescriptor =
316        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("foo")).build();
317      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
318      admin.createTable(tableDescriptorBuilder.build());
319      assertEquals(1, createTableTimer.getHistogram().getCount() - prevCount);
320    }
321  }
322
323  @Test
324  public void testRegionServerObserver() throws IOException {
325    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
326      Admin admin = connection.getAdmin()) {
327      LOG.info("Rolling WALs");
328      admin.rollWALWriter(UTIL.getMiniHBaseCluster().getServerHoldingMeta());
329    }
330
331    // Find out the MetricRegistry used by the CP using the global registries
332    MetricRegistryInfo info = MetricsCoprocessor
333      .createRegistryInfoForRSCoprocessor(CustomRegionServerObserver.class.getName());
334
335    Optional<MetricRegistry> registry = MetricRegistries.global().get(info);
336    assertTrue(registry.isPresent());
337
338    Optional<Metric> metric = registry.get().get("rollWALRequests");
339    assertTrue(metric.isPresent());
340
341    Counter rollWalRequests = (Counter) metric.get();
342    assertEquals(1, rollWalRequests.getCount());
343  }
344
345  @Test
346  public void testWALObserver() throws IOException {
347    // Find out the MetricRegistry used by the CP using the global registries
348    MetricRegistryInfo info =
349      MetricsCoprocessor.createRegistryInfoForWALCoprocessor(CustomWALObserver.class.getName());
350
351    Optional<MetricRegistry> registry = MetricRegistries.global().get(info);
352    assertTrue(registry.isPresent());
353
354    Optional<Metric> metric = registry.get().get("walEditsCount");
355    assertTrue(metric.isPresent());
356
357    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
358      Admin admin = connection.getAdmin()) {
359      TableDescriptorBuilder tableDescriptorBuilder =
360        TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
361      ColumnFamilyDescriptor columnFamilyDescriptor =
362        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("foo")).build();
363      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
364      admin.createTable(tableDescriptorBuilder.build());
365
366      Counter rollWalRequests = (Counter) metric.get();
367      long prevCount = rollWalRequests.getCount();
368      assertTrue(prevCount > 0);
369
370      try (Table table = connection.getTable(TableName.valueOf(name.getMethodName()))) {
371        table.put(new Put(foo).addColumn(foo, foo, foo));
372      }
373
374      assertEquals(1, rollWalRequests.getCount() - prevCount);
375    }
376  }
377
378  /**
379   * Helper for below tests
380   */
381  private void assertPreGetRequestsCounter(Class<?> coprocClass) {
382    // Find out the MetricRegistry used by the CP using the global registries
383    MetricRegistryInfo info =
384      MetricsCoprocessor.createRegistryInfoForRegionCoprocessor(coprocClass.getName());
385
386    Optional<MetricRegistry> registry = MetricRegistries.global().get(info);
387    assertTrue(registry.isPresent());
388
389    Optional<Metric> metric = registry.get().get("preGetRequests");
390    assertTrue(metric.isPresent());
391
392    Counter preGetRequests = (Counter) metric.get();
393    assertEquals(2, preGetRequests.getCount());
394  }
395
396  @Test
397  public void testRegionObserverSingleRegion() throws IOException {
398    final TableName tableName = TableName.valueOf(name.getMethodName());
399    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
400      Admin admin = connection.getAdmin()) {
401      admin.createTable(TableDescriptorBuilder.newBuilder(tableName)
402        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
403        // add the coprocessor for the region
404        .setCoprocessor(CustomRegionObserver.class.getName()).build());
405      try (Table table = connection.getTable(tableName)) {
406        table.get(new Get(foo));
407        table.get(new Get(foo)); // 2 gets
408      }
409    }
410
411    assertPreGetRequestsCounter(CustomRegionObserver.class);
412  }
413
414  @Test
415  public void testRegionObserverMultiRegion() throws IOException {
416    final TableName tableName = TableName.valueOf(name.getMethodName());
417    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
418      Admin admin = connection.getAdmin()) {
419      admin.createTable(TableDescriptorBuilder.newBuilder(tableName)
420        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
421        // add the coprocessor for the region
422        .setCoprocessor(CustomRegionObserver.class.getName()).build(), new byte[][] { foo });
423      // create with 2 regions
424      try (Table table = connection.getTable(tableName);
425        RegionLocator locator = connection.getRegionLocator(tableName)) {
426        table.get(new Get(bar));
427        table.get(new Get(foo)); // 2 gets to 2 separate regions
428        assertEquals(2, locator.getAllRegionLocations().size());
429        assertNotEquals(locator.getRegionLocation(bar).getRegion(),
430          locator.getRegionLocation(foo).getRegion());
431      }
432    }
433
434    assertPreGetRequestsCounter(CustomRegionObserver.class);
435  }
436
437  @Test
438  public void testRegionObserverMultiTable() throws IOException {
439    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
440    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
441    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
442      Admin admin = connection.getAdmin()) {
443      admin.createTable(TableDescriptorBuilder.newBuilder(tableName1)
444        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
445        // add the coprocessor for the region
446        .setCoprocessor(CustomRegionObserver.class.getName()).build());
447      admin.createTable(TableDescriptorBuilder.newBuilder(tableName2)
448        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
449        // add the coprocessor for the region
450        .setCoprocessor(CustomRegionObserver.class.getName()).build());
451      try (Table table1 = connection.getTable(tableName1);
452        Table table2 = connection.getTable(tableName2)) {
453        table1.get(new Get(bar));
454        table2.get(new Get(foo)); // 2 gets to 2 separate tables
455      }
456    }
457    assertPreGetRequestsCounter(CustomRegionObserver.class);
458  }
459
460  @Test
461  public void testRegionObserverMultiCoprocessor() throws IOException {
462    final TableName tableName = TableName.valueOf(name.getMethodName());
463    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
464      Admin admin = connection.getAdmin()) {
465      admin.createTable(TableDescriptorBuilder.newBuilder(tableName)
466        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
467        // add the coprocessor for the region. We add two different coprocessors
468        .setCoprocessor(CustomRegionObserver.class.getName())
469        .setCoprocessor(CustomRegionObserver2.class.getName()).build());
470      try (Table table = connection.getTable(tableName)) {
471        table.get(new Get(foo));
472        table.get(new Get(foo)); // 2 gets
473      }
474    }
475
476    // we will have two counters coming from two coprocs, in two different MetricRegistries
477    assertPreGetRequestsCounter(CustomRegionObserver.class);
478    assertPreGetRequestsCounter(CustomRegionObserver2.class);
479  }
480
481  @Test
482  public void testRegionObserverAfterRegionClosed() throws IOException {
483    final TableName tableName = TableName.valueOf(name.getMethodName());
484    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
485      Admin admin = connection.getAdmin()) {
486      admin.createTable(TableDescriptorBuilder.newBuilder(tableName)
487        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
488        // add the coprocessor for the region
489        .setCoprocessor(CustomRegionObserver.class.getName()).build(), new byte[][] { foo });
490      // create with 2 regions
491      try (Table table = connection.getTable(tableName)) {
492        table.get(new Get(foo));
493        table.get(new Get(foo)); // 2 gets
494      }
495
496      assertPreGetRequestsCounter(CustomRegionObserver.class);
497
498      // close one of the regions
499      try (RegionLocator locator = connection.getRegionLocator(tableName)) {
500        HRegionLocation loc = locator.getRegionLocation(foo);
501        admin.unassign(loc.getRegion().getEncodedNameAsBytes(), true);
502
503        HRegionServer server = UTIL.getMiniHBaseCluster().getRegionServer(loc.getServerName());
504        UTIL.waitFor(30000, () -> server.getOnlineRegion(loc.getRegion().getRegionName()) == null);
505        assertNull(server.getOnlineRegion(loc.getRegion().getRegionName()));
506      }
507
508      // with only 1 region remaining, we should still be able to find the Counter
509      assertPreGetRequestsCounter(CustomRegionObserver.class);
510
511      // close the table
512      admin.disableTable(tableName);
513
514      MetricRegistryInfo info = MetricsCoprocessor
515        .createRegistryInfoForRegionCoprocessor(CustomRegionObserver.class.getName());
516
517      // ensure that MetricRegistry is deleted
518      Optional<MetricRegistry> registry = MetricRegistries.global().get(info);
519      assertFalse(registry.isPresent());
520    }
521  }
522
523  @Test
524  public void testRegionObserverEndpoint() throws IOException, ServiceException {
525    final TableName tableName = TableName.valueOf(name.getMethodName());
526    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
527      Admin admin = connection.getAdmin()) {
528      admin.createTable(TableDescriptorBuilder.newBuilder(tableName)
529        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
530        // add the coprocessor for the region
531        .setCoprocessor(CustomRegionEndpoint.class.getName()).build());
532
533      try (Table table = connection.getTable(tableName)) {
534        List<Mutation> mutations = Lists.newArrayList(new Put(foo), new Put(bar));
535        MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
536
537        for (Mutation mutation : mutations) {
538          mrmBuilder.addMutationRequest(
539            ProtobufUtil.toMutation(ClientProtos.MutationProto.MutationType.PUT, mutation));
540        }
541
542        CoprocessorRpcChannel channel = table.coprocessorService(bar);
543        MultiRowMutationService.BlockingInterface service =
544          MultiRowMutationService.newBlockingStub(channel);
545        MutateRowsRequest mrm = mrmBuilder.build();
546        service.mutateRows(null, mrm);
547      }
548    }
549
550    // Find out the MetricRegistry used by the CP using the global registries
551    MetricRegistryInfo info = MetricsCoprocessor
552      .createRegistryInfoForRegionCoprocessor(CustomRegionEndpoint.class.getName());
553
554    Optional<MetricRegistry> registry = MetricRegistries.global().get(info);
555    assertTrue(registry.isPresent());
556
557    Optional<Metric> metric = registry.get().get("EndpointExecution");
558    assertTrue(metric.isPresent());
559
560    Timer endpointExecutions = (Timer) metric.get();
561    assertEquals(1, endpointExecutions.getHistogram().getCount());
562  }
563}