001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.coprocessor;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025
026import java.io.IOException;
027import java.util.List;
028import java.util.Optional;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.CoprocessorEnvironment;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtil;
034import org.apache.hadoop.hbase.HRegionLocation;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.Admin;
037import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
038import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
039import org.apache.hadoop.hbase.client.Connection;
040import org.apache.hadoop.hbase.client.ConnectionFactory;
041import org.apache.hadoop.hbase.client.Get;
042import org.apache.hadoop.hbase.client.Mutation;
043import org.apache.hadoop.hbase.client.Put;
044import org.apache.hadoop.hbase.client.RegionInfo;
045import org.apache.hadoop.hbase.client.RegionLocator;
046import org.apache.hadoop.hbase.client.Table;
047import org.apache.hadoop.hbase.client.TableDescriptor;
048import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
049import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
050import org.apache.hadoop.hbase.metrics.Counter;
051import org.apache.hadoop.hbase.metrics.Metric;
052import org.apache.hadoop.hbase.metrics.MetricRegistries;
053import org.apache.hadoop.hbase.metrics.MetricRegistry;
054import org.apache.hadoop.hbase.metrics.MetricRegistryInfo;
055import org.apache.hadoop.hbase.metrics.Timer;
056import org.apache.hadoop.hbase.regionserver.HRegionServer;
057import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
058import org.apache.hadoop.hbase.testclassification.MediumTests;
059import org.apache.hadoop.hbase.util.Bytes;
060import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
061import org.apache.hadoop.hbase.wal.WALEdit;
062import org.apache.hadoop.hbase.wal.WALKey;
063import org.junit.AfterClass;
064import org.junit.Before;
065import org.junit.BeforeClass;
066import org.junit.ClassRule;
067import org.junit.Rule;
068import org.junit.Test;
069import org.junit.experimental.categories.Category;
070import org.junit.rules.TestName;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
075import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
076import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
077import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
078
079import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
080import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
081import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
082import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
083import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MutateRowsResponse;
084
085/**
086 * Testing of coprocessor metrics end-to-end.
087 */
088@Category({CoprocessorTests.class, MediumTests.class})
089public class TestCoprocessorMetrics {
090
091  @ClassRule
092  public static final HBaseClassTestRule CLASS_RULE =
093      HBaseClassTestRule.forClass(TestCoprocessorMetrics.class);
094
095  private static final Logger LOG = LoggerFactory.getLogger(TestCoprocessorMetrics.class);
096  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
097
098  private static final byte[] foo = Bytes.toBytes("foo");
099  private static final byte[] bar = Bytes.toBytes("bar");
100
101  @Rule
102  public TestName name = new TestName();
103
104  /**
105   * MasterObserver that has a Timer metric for create table operation.
106   */
107  public static class CustomMasterObserver implements MasterCoprocessor, MasterObserver {
108    private Timer createTableTimer;
109    private long start = Long.MIN_VALUE;
110
111    @Override
112    public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
113                               TableDescriptor desc, RegionInfo[] regions) throws IOException {
114      // we rely on the fact that there is only 1 instance of our MasterObserver
115      this.start = EnvironmentEdgeManager.currentTime();
116    }
117
118    @Override
119    public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
120                                TableDescriptor desc, RegionInfo[] regions) throws IOException {
121      if (this.start > 0) {
122        long time = EnvironmentEdgeManager.currentTime() - start;
123        LOG.info("Create table took: " + time);
124        createTableTimer.updateMillis(time);
125      }
126    }
127
128    @Override
129    public void start(CoprocessorEnvironment env) throws IOException {
130      if (env instanceof MasterCoprocessorEnvironment) {
131        MetricRegistry registry =
132            ((MasterCoprocessorEnvironment) env).getMetricRegistryForMaster();
133
134        createTableTimer  = registry.timer("CreateTable");
135      }
136    }
137
138    @Override
139    public Optional<MasterObserver> getMasterObserver() {
140      return Optional.of(this);
141    }
142  }
143
144  /**
145   * RegionServerObserver that has a Counter for rollWAL requests.
146   */
147  public static class CustomRegionServerObserver implements RegionServerCoprocessor,
148      RegionServerObserver {
149    /** This is the Counter metric object to keep track of the current count across invocations */
150    private Counter rollWALCounter;
151
152    @Override public Optional<RegionServerObserver> getRegionServerObserver() {
153      return Optional.of(this);
154    }
155
156    @Override
157    public void postRollWALWriterRequest(ObserverContext<RegionServerCoprocessorEnvironment> ctx)
158        throws IOException {
159      // Increment the Counter whenever the coprocessor is called
160      rollWALCounter.increment();
161    }
162
163    @Override
164    public void start(CoprocessorEnvironment env) throws IOException {
165      if (env instanceof RegionServerCoprocessorEnvironment) {
166        MetricRegistry registry =
167            ((RegionServerCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
168
169        if (rollWALCounter == null) {
170          rollWALCounter = registry.counter("rollWALRequests");
171        }
172      }
173    }
174  }
175
176  /**
177   * WALObserver that has a Counter for walEdits written.
178   */
179  public static class CustomWALObserver implements WALCoprocessor, WALObserver {
180    private Counter walEditsCount;
181
182    @Override
183    public void postWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
184                             RegionInfo info, WALKey logKey,
185                             WALEdit logEdit) throws IOException {
186      walEditsCount.increment();
187    }
188
189    @Override
190    public void start(CoprocessorEnvironment env) throws IOException {
191      if (env instanceof WALCoprocessorEnvironment) {
192        MetricRegistry registry =
193            ((WALCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
194
195        if (walEditsCount == null) {
196          walEditsCount = registry.counter("walEditsCount");
197        }
198      }
199    }
200
201    @Override public Optional<WALObserver> getWALObserver() {
202      return Optional.of(this);
203    }
204  }
205
206  /**
207   * RegionObserver that has a Counter for preGet()
208   */
209  public static class CustomRegionObserver implements RegionCoprocessor, RegionObserver {
210    private Counter preGetCounter;
211
212    @Override
213    public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get,
214                         List<Cell> results) throws IOException {
215      preGetCounter.increment();
216    }
217
218    @Override
219    public Optional<RegionObserver> getRegionObserver() {
220      return Optional.of(this);
221    }
222
223    @Override
224    public void start(CoprocessorEnvironment env) throws IOException {
225      if (env instanceof RegionCoprocessorEnvironment) {
226        MetricRegistry registry =
227            ((RegionCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
228
229        if (preGetCounter == null) {
230          preGetCounter = registry.counter("preGetRequests");
231        }
232      }
233    }
234  }
235
236  public static class CustomRegionObserver2 extends CustomRegionObserver {
237  }
238
239  /**
240   * RegionEndpoint to test metrics from endpoint calls
241   */
242  public static class CustomRegionEndpoint extends MultiRowMutationEndpoint {
243
244    private Timer endpointExecution;
245
246    @Override
247    public void mutateRows(RpcController controller, MutateRowsRequest request,
248                           RpcCallback<MutateRowsResponse> done) {
249      long start = System.nanoTime();
250      super.mutateRows(controller, request, done);
251      endpointExecution.updateNanos(System.nanoTime() - start);
252    }
253
254    @Override
255    public void start(CoprocessorEnvironment env) throws IOException {
256      super.start(env);
257
258      if (env instanceof RegionCoprocessorEnvironment) {
259        MetricRegistry registry =
260            ((RegionCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
261
262        if (endpointExecution == null) {
263          endpointExecution = registry.timer("EndpointExecution");
264        }
265      }
266    }
267  }
268
269  @BeforeClass
270  public static void setupBeforeClass() throws Exception {
271    Configuration conf = UTIL.getConfiguration();
272    // inject master, regionserver and WAL coprocessors
273    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
274        CustomMasterObserver.class.getName());
275    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
276        CustomRegionServerObserver.class.getName());
277    conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
278        CustomWALObserver.class.getName());
279    conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
280    UTIL.startMiniCluster();
281  }
282
283  @AfterClass
284  public static void teardownAfterClass() throws Exception {
285    UTIL.shutdownMiniCluster();
286  }
287
288  @Before
289  public void setup() throws IOException {
290    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
291         Admin admin = connection.getAdmin()) {
292      for (TableDescriptor htd : admin.listTableDescriptors()) {
293        UTIL.deleteTable(htd.getTableName());
294      }
295    }
296  }
297
298  @Test
299  public void testMasterObserver() throws IOException {
300    // Find out the MetricRegistry used by the CP using the global registries
301    MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForMasterCoprocessor(
302        CustomMasterObserver.class.getName());
303    Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
304    assertTrue(registry.isPresent());
305
306    Optional<Metric> metric = registry.get().get("CreateTable");
307    assertTrue(metric.isPresent());
308
309    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
310         Admin admin = connection.getAdmin()) {
311
312      Timer createTableTimer = (Timer)metric.get();
313      long prevCount = createTableTimer.getHistogram().getCount();
314      LOG.info("Creating table");
315      TableDescriptorBuilder tableDescriptorBuilder =
316        TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
317      ColumnFamilyDescriptor columnFamilyDescriptor =
318        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("foo")).build();
319      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
320      admin.createTable(tableDescriptorBuilder.build());
321      assertEquals(1, createTableTimer.getHistogram().getCount() - prevCount);
322    }
323  }
324
325  @Test
326  public void testRegionServerObserver() throws IOException {
327    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
328         Admin admin = connection.getAdmin()) {
329      LOG.info("Rolling WALs");
330      admin.rollWALWriter(UTIL.getMiniHBaseCluster().getServerHoldingMeta());
331    }
332
333    // Find out the MetricRegistry used by the CP using the global registries
334    MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRSCoprocessor(
335        CustomRegionServerObserver.class.getName());
336
337    Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
338    assertTrue(registry.isPresent());
339
340    Optional<Metric> metric = registry.get().get("rollWALRequests");
341    assertTrue(metric.isPresent());
342
343    Counter rollWalRequests = (Counter)metric.get();
344    assertEquals(1, rollWalRequests.getCount());
345  }
346
347  @Test
348  public void testWALObserver() throws IOException {
349    // Find out the MetricRegistry used by the CP using the global registries
350    MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForWALCoprocessor(
351        CustomWALObserver.class.getName());
352
353    Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
354    assertTrue(registry.isPresent());
355
356    Optional<Metric> metric = registry.get().get("walEditsCount");
357    assertTrue(metric.isPresent());
358
359    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
360         Admin admin = connection.getAdmin()) {
361      TableDescriptorBuilder tableDescriptorBuilder =
362        TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
363      ColumnFamilyDescriptor columnFamilyDescriptor =
364        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("foo")).build();
365      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
366      admin.createTable(tableDescriptorBuilder.build());
367
368      Counter rollWalRequests = (Counter)metric.get();
369      long prevCount = rollWalRequests.getCount();
370      assertTrue(prevCount > 0);
371
372      try (Table table = connection.getTable(TableName.valueOf(name.getMethodName()))) {
373        table.put(new Put(foo).addColumn(foo, foo, foo));
374      }
375
376      assertEquals(1, rollWalRequests.getCount() - prevCount);
377    }
378  }
379
380  /**
381   * Helper for below tests
382   */
383  private void assertPreGetRequestsCounter(Class<?> coprocClass) {
384    // Find out the MetricRegistry used by the CP using the global registries
385    MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRegionCoprocessor(
386        coprocClass.getName());
387
388    Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
389    assertTrue(registry.isPresent());
390
391    Optional<Metric> metric = registry.get().get("preGetRequests");
392    assertTrue(metric.isPresent());
393
394    Counter preGetRequests = (Counter)metric.get();
395    assertEquals(2, preGetRequests.getCount());
396  }
397
398  @Test
399  public void testRegionObserverSingleRegion() throws IOException {
400    final TableName tableName = TableName.valueOf(name.getMethodName());
401    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
402      Admin admin = connection.getAdmin()) {
403      admin.createTable(TableDescriptorBuilder.newBuilder(tableName)
404        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
405        // add the coprocessor for the region
406        .setCoprocessor(CustomRegionObserver.class.getName()).build());
407      try (Table table = connection.getTable(tableName)) {
408        table.get(new Get(foo));
409        table.get(new Get(foo)); // 2 gets
410      }
411    }
412
413    assertPreGetRequestsCounter(CustomRegionObserver.class);
414  }
415
416  @Test
417  public void testRegionObserverMultiRegion() throws IOException {
418    final TableName tableName = TableName.valueOf(name.getMethodName());
419    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
420         Admin admin = connection.getAdmin()) {
421      admin.createTable(TableDescriptorBuilder.newBuilder(tableName)
422        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
423        // add the coprocessor for the region
424        .setCoprocessor(CustomRegionObserver.class.getName()).build(), new byte[][] { foo });
425      // create with 2 regions
426      try (Table table = connection.getTable(tableName);
427           RegionLocator locator = connection.getRegionLocator(tableName)) {
428        table.get(new Get(bar));
429        table.get(new Get(foo)); // 2 gets to 2 separate regions
430        assertEquals(2, locator.getAllRegionLocations().size());
431        assertNotEquals(locator.getRegionLocation(bar).getRegion(),
432            locator.getRegionLocation(foo).getRegion());
433      }
434    }
435
436    assertPreGetRequestsCounter(CustomRegionObserver.class);
437  }
438
439  @Test
440  public void testRegionObserverMultiTable() throws IOException {
441    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
442    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
443    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
444         Admin admin = connection.getAdmin()) {
445      admin.createTable(TableDescriptorBuilder.newBuilder(tableName1)
446        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
447        // add the coprocessor for the region
448        .setCoprocessor(CustomRegionObserver.class.getName()).build());
449      admin.createTable(TableDescriptorBuilder.newBuilder(tableName2)
450        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
451        // add the coprocessor for the region
452        .setCoprocessor(CustomRegionObserver.class.getName()).build());
453      try (Table table1 = connection.getTable(tableName1);
454           Table table2 = connection.getTable(tableName2)) {
455        table1.get(new Get(bar));
456        table2.get(new Get(foo)); // 2 gets to 2 separate tables
457      }
458    }
459    assertPreGetRequestsCounter(CustomRegionObserver.class);
460  }
461
462  @Test
463  public void testRegionObserverMultiCoprocessor() throws IOException {
464    final TableName tableName = TableName.valueOf(name.getMethodName());
465    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
466         Admin admin = connection.getAdmin()) {
467      admin.createTable(TableDescriptorBuilder.newBuilder(tableName)
468        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
469        // add the coprocessor for the region. We add two different coprocessors
470        .setCoprocessor(CustomRegionObserver.class.getName())
471        .setCoprocessor(CustomRegionObserver2.class.getName()).build());
472      try (Table table = connection.getTable(tableName)) {
473        table.get(new Get(foo));
474        table.get(new Get(foo)); // 2 gets
475      }
476    }
477
478    // we will have two counters coming from two coprocs, in two different MetricRegistries
479    assertPreGetRequestsCounter(CustomRegionObserver.class);
480    assertPreGetRequestsCounter(CustomRegionObserver2.class);
481  }
482
483  @Test
484  public void testRegionObserverAfterRegionClosed() throws IOException {
485    final TableName tableName = TableName.valueOf(name.getMethodName());
486    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
487         Admin admin = connection.getAdmin()) {
488      admin.createTable(TableDescriptorBuilder.newBuilder(tableName)
489        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
490        // add the coprocessor for the region
491        .setCoprocessor(CustomRegionObserver.class.getName()).build(), new byte[][] { foo });
492      // create with 2 regions
493      try (Table table = connection.getTable(tableName)) {
494        table.get(new Get(foo));
495        table.get(new Get(foo)); // 2 gets
496      }
497
498      assertPreGetRequestsCounter(CustomRegionObserver.class);
499
500      // close one of the regions
501      try (RegionLocator locator = connection.getRegionLocator(tableName)) {
502        HRegionLocation loc = locator.getRegionLocation(foo);
503        admin.unassign(loc.getRegion().getEncodedNameAsBytes(), true);
504
505        HRegionServer server = UTIL.getMiniHBaseCluster().getRegionServer(loc.getServerName());
506        UTIL.waitFor(30000,
507          () -> server.getOnlineRegion(loc.getRegion().getRegionName()) == null);
508        assertNull(server.getOnlineRegion(loc.getRegion().getRegionName()));
509      }
510
511      // with only 1 region remaining, we should still be able to find the Counter
512      assertPreGetRequestsCounter(CustomRegionObserver.class);
513
514      // close the table
515      admin.disableTable(tableName);
516
517      MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRegionCoprocessor(
518          CustomRegionObserver.class.getName());
519
520      // ensure that MetricRegistry is deleted
521      Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
522      assertFalse(registry.isPresent());
523    }
524  }
525
526  @Test
527  public void testRegionObserverEndpoint() throws IOException, ServiceException {
528    final TableName tableName = TableName.valueOf(name.getMethodName());
529    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
530         Admin admin = connection.getAdmin()) {
531      admin.createTable(TableDescriptorBuilder.newBuilder(tableName)
532        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo))
533        // add the coprocessor for the region
534        .setCoprocessor(CustomRegionEndpoint.class.getName()).build());
535
536      try (Table table = connection.getTable(tableName)) {
537        List<Mutation> mutations = Lists.newArrayList(new Put(foo), new Put(bar));
538        MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
539
540        for (Mutation mutation : mutations) {
541          mrmBuilder.addMutationRequest(ProtobufUtil.toMutation(
542              ClientProtos.MutationProto.MutationType.PUT, mutation));
543        }
544
545        CoprocessorRpcChannel channel = table.coprocessorService(bar);
546        MultiRowMutationService.BlockingInterface service =
547            MultiRowMutationService.newBlockingStub(channel);
548        MutateRowsRequest mrm = mrmBuilder.build();
549        service.mutateRows(null, mrm);
550      }
551    }
552
553    // Find out the MetricRegistry used by the CP using the global registries
554    MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRegionCoprocessor(
555        CustomRegionEndpoint.class.getName());
556
557    Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
558    assertTrue(registry.isPresent());
559
560    Optional<Metric> metric = registry.get().get("EndpointExecution");
561    assertTrue(metric.isPresent());
562
563    Timer endpointExecutions = (Timer)metric.get();
564    assertEquals(1, endpointExecutions.getHistogram().getCount());
565  }
566}