001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.coprocessor;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025
026import com.google.protobuf.RpcCallback;
027import com.google.protobuf.RpcController;
028import com.google.protobuf.ServiceException;
029import java.io.IOException;
030import java.util.List;
031import java.util.Optional;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.hbase.Cell;
034import org.apache.hadoop.hbase.CoprocessorEnvironment;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtility;
037import org.apache.hadoop.hbase.HColumnDescriptor;
038import org.apache.hadoop.hbase.HRegionLocation;
039import org.apache.hadoop.hbase.HTableDescriptor;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.client.Admin;
042import org.apache.hadoop.hbase.client.Connection;
043import org.apache.hadoop.hbase.client.ConnectionFactory;
044import org.apache.hadoop.hbase.client.Get;
045import org.apache.hadoop.hbase.client.Mutation;
046import org.apache.hadoop.hbase.client.Put;
047import org.apache.hadoop.hbase.client.RegionInfo;
048import org.apache.hadoop.hbase.client.RegionLocator;
049import org.apache.hadoop.hbase.client.Table;
050import org.apache.hadoop.hbase.client.TableDescriptor;
051import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
052import org.apache.hadoop.hbase.metrics.Counter;
053import org.apache.hadoop.hbase.metrics.Metric;
054import org.apache.hadoop.hbase.metrics.MetricRegistries;
055import org.apache.hadoop.hbase.metrics.MetricRegistry;
056import org.apache.hadoop.hbase.metrics.MetricRegistryInfo;
057import org.apache.hadoop.hbase.metrics.Timer;
058import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
059import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
060import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
061import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
062import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsResponse;
063import org.apache.hadoop.hbase.regionserver.HRegionServer;
064import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
065import org.apache.hadoop.hbase.testclassification.MediumTests;
066import org.apache.hadoop.hbase.util.Bytes;
067import org.apache.hadoop.hbase.wal.WALEdit;
068import org.apache.hadoop.hbase.wal.WALKey;
069import org.junit.AfterClass;
070import org.junit.Before;
071import org.junit.BeforeClass;
072import org.junit.ClassRule;
073import org.junit.Rule;
074import org.junit.Test;
075import org.junit.experimental.categories.Category;
076import org.junit.rules.TestName;
077import org.slf4j.Logger;
078import org.slf4j.LoggerFactory;
079
080import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
081
082/**
083 * Testing of coprocessor metrics end-to-end.
084 */
085@Category({CoprocessorTests.class, MediumTests.class})
086public class TestCoprocessorMetrics {
087
088  @ClassRule
089  public static final HBaseClassTestRule CLASS_RULE =
090      HBaseClassTestRule.forClass(TestCoprocessorMetrics.class);
091
092  private static final Logger LOG = LoggerFactory.getLogger(TestCoprocessorMetrics.class);
093  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
094
095  private static final byte[] foo = Bytes.toBytes("foo");
096  private static final byte[] bar = Bytes.toBytes("bar");
097
098  @Rule
099  public TestName name = new TestName();
100
101  /**
102   * MasterObserver that has a Timer metric for create table operation.
103   */
104  public static class CustomMasterObserver implements MasterCoprocessor, MasterObserver {
105    private Timer createTableTimer;
106    private long start = Long.MIN_VALUE;
107
108    @Override
109    public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
110                               TableDescriptor desc, RegionInfo[] regions) throws IOException {
111      // we rely on the fact that there is only 1 instance of our MasterObserver
112      this.start = System.currentTimeMillis();
113    }
114
115    @Override
116    public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
117                                TableDescriptor desc, RegionInfo[] regions) throws IOException {
118      if (this.start > 0) {
119        long time = System.currentTimeMillis() - start;
120        LOG.info("Create table took: " + time);
121        createTableTimer.updateMillis(time);
122      }
123    }
124
125    @Override
126    public void start(CoprocessorEnvironment env) throws IOException {
127      if (env instanceof MasterCoprocessorEnvironment) {
128        MetricRegistry registry =
129            ((MasterCoprocessorEnvironment) env).getMetricRegistryForMaster();
130
131        createTableTimer  = registry.timer("CreateTable");
132      }
133    }
134
135    @Override
136    public Optional<MasterObserver> getMasterObserver() {
137      return Optional.of(this);
138    }
139  }
140
141  /**
142   * RegionServerObserver that has a Counter for rollWAL requests.
143   */
144  public static class CustomRegionServerObserver implements RegionServerCoprocessor,
145      RegionServerObserver {
146    /** This is the Counter metric object to keep track of the current count across invocations */
147    private Counter rollWALCounter;
148
149    @Override public Optional<RegionServerObserver> getRegionServerObserver() {
150      return Optional.of(this);
151    }
152
153    @Override
154    public void postRollWALWriterRequest(ObserverContext<RegionServerCoprocessorEnvironment> ctx)
155        throws IOException {
156      // Increment the Counter whenever the coprocessor is called
157      rollWALCounter.increment();
158    }
159
160    @Override
161    public void start(CoprocessorEnvironment env) throws IOException {
162      if (env instanceof RegionServerCoprocessorEnvironment) {
163        MetricRegistry registry =
164            ((RegionServerCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
165
166        if (rollWALCounter == null) {
167          rollWALCounter = registry.counter("rollWALRequests");
168        }
169      }
170    }
171  }
172
173  /**
174   * WALObserver that has a Counter for walEdits written.
175   */
176  public static class CustomWALObserver implements WALCoprocessor, WALObserver {
177    private Counter walEditsCount;
178
179    @Override
180    public void postWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
181                             RegionInfo info, WALKey logKey,
182                             WALEdit logEdit) throws IOException {
183      walEditsCount.increment();
184    }
185
186    @Override
187    public void start(CoprocessorEnvironment env) throws IOException {
188      if (env instanceof WALCoprocessorEnvironment) {
189        MetricRegistry registry =
190            ((WALCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
191
192        if (walEditsCount == null) {
193          walEditsCount = registry.counter("walEditsCount");
194        }
195      }
196    }
197
198    @Override public Optional<WALObserver> getWALObserver() {
199      return Optional.of(this);
200    }
201  }
202
203  /**
204   * RegionObserver that has a Counter for preGet()
205   */
206  public static class CustomRegionObserver implements RegionCoprocessor, RegionObserver {
207    private Counter preGetCounter;
208
209    @Override
210    public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get,
211                         List<Cell> results) throws IOException {
212      preGetCounter.increment();
213    }
214
215    @Override
216    public Optional<RegionObserver> getRegionObserver() {
217      return Optional.of(this);
218    }
219
220    @Override
221    public void start(CoprocessorEnvironment env) throws IOException {
222      if (env instanceof RegionCoprocessorEnvironment) {
223        MetricRegistry registry =
224            ((RegionCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
225
226        if (preGetCounter == null) {
227          preGetCounter = registry.counter("preGetRequests");
228        }
229      }
230    }
231  }
232
233  public static class CustomRegionObserver2 extends CustomRegionObserver {
234  }
235
236  /**
237   * RegionEndpoint to test metrics from endpoint calls
238   */
239  public static class CustomRegionEndpoint extends MultiRowMutationEndpoint {
240
241    private Timer endpointExecution;
242
243    @Override
244    public void mutateRows(RpcController controller, MutateRowsRequest request,
245                           RpcCallback<MutateRowsResponse> done) {
246      long start = System.nanoTime();
247      super.mutateRows(controller, request, done);
248      endpointExecution.updateNanos(System.nanoTime() - start);
249    }
250
251    @Override
252    public void start(CoprocessorEnvironment env) throws IOException {
253      super.start(env);
254
255      if (env instanceof RegionCoprocessorEnvironment) {
256        MetricRegistry registry =
257            ((RegionCoprocessorEnvironment) env).getMetricRegistryForRegionServer();
258
259        if (endpointExecution == null) {
260          endpointExecution = registry.timer("EndpointExecution");
261        }
262      }
263    }
264  }
265
266  @BeforeClass
267  public static void setupBeforeClass() throws Exception {
268    Configuration conf = UTIL.getConfiguration();
269    // inject master, regionserver and WAL coprocessors
270    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
271        CustomMasterObserver.class.getName());
272    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
273        CustomRegionServerObserver.class.getName());
274    conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
275        CustomWALObserver.class.getName());
276    conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
277    UTIL.startMiniCluster();
278  }
279
280  @AfterClass
281  public static void teardownAfterClass() throws Exception {
282    UTIL.shutdownMiniCluster();
283  }
284
285  @Before
286  public void setup() throws IOException {
287    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
288         Admin admin = connection.getAdmin()) {
289      for (HTableDescriptor htd : admin.listTables()) {
290        UTIL.deleteTable(htd.getTableName());
291      }
292    }
293  }
294
295  @Test
296  public void testMasterObserver() throws IOException {
297    // Find out the MetricRegistry used by the CP using the global registries
298    MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForMasterCoprocessor(
299        CustomMasterObserver.class.getName());
300    Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
301    assertTrue(registry.isPresent());
302
303    Optional<Metric> metric = registry.get().get("CreateTable");
304    assertTrue(metric.isPresent());
305
306    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
307         Admin admin = connection.getAdmin()) {
308
309      Timer createTableTimer = (Timer)metric.get();
310      long prevCount = createTableTimer.getHistogram().getCount();
311      LOG.info("Creating table");
312      admin.createTable(
313          new HTableDescriptor(TableName.valueOf(name.getMethodName()))
314              .addFamily(new HColumnDescriptor("foo")));
315
316      assertEquals(1, createTableTimer.getHistogram().getCount() - prevCount);
317    }
318  }
319
320  @Test
321  public void testRegionServerObserver() throws IOException {
322    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
323         Admin admin = connection.getAdmin()) {
324      LOG.info("Rolling WALs");
325      admin.rollWALWriter(UTIL.getMiniHBaseCluster().getServerHoldingMeta());
326    }
327
328    // Find out the MetricRegistry used by the CP using the global registries
329    MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRSCoprocessor(
330        CustomRegionServerObserver.class.getName());
331
332    Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
333    assertTrue(registry.isPresent());
334
335    Optional<Metric> metric = registry.get().get("rollWALRequests");
336    assertTrue(metric.isPresent());
337
338    Counter rollWalRequests = (Counter)metric.get();
339    assertEquals(1, rollWalRequests.getCount());
340  }
341
342  @Test
343  public void testWALObserver() throws IOException {
344    // Find out the MetricRegistry used by the CP using the global registries
345    MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForWALCoprocessor(
346        CustomWALObserver.class.getName());
347
348    Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
349    assertTrue(registry.isPresent());
350
351    Optional<Metric> metric = registry.get().get("walEditsCount");
352    assertTrue(metric.isPresent());
353
354    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
355         Admin admin = connection.getAdmin()) {
356      admin.createTable(
357          new HTableDescriptor(TableName.valueOf(name.getMethodName()))
358              .addFamily(new HColumnDescriptor("foo")));
359
360      Counter rollWalRequests = (Counter)metric.get();
361      long prevCount = rollWalRequests.getCount();
362      assertTrue(prevCount > 0);
363
364      try (Table table = connection.getTable(TableName.valueOf(name.getMethodName()))) {
365        table.put(new Put(foo).addColumn(foo, foo, foo));
366      }
367
368      assertEquals(1, rollWalRequests.getCount() - prevCount);
369    }
370  }
371
372  /**
373   * Helper for below tests
374   */
375  private void assertPreGetRequestsCounter(Class<?> coprocClass) {
376    // Find out the MetricRegistry used by the CP using the global registries
377    MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRegionCoprocessor(
378        coprocClass.getName());
379
380    Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
381    assertTrue(registry.isPresent());
382
383    Optional<Metric> metric = registry.get().get("preGetRequests");
384    assertTrue(metric.isPresent());
385
386    Counter preGetRequests = (Counter)metric.get();
387    assertEquals(2, preGetRequests.getCount());
388  }
389
390  @Test
391  public void testRegionObserverSingleRegion() throws IOException {
392    final TableName tableName = TableName.valueOf(name.getMethodName());
393    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
394         Admin admin = connection.getAdmin()) {
395      admin.createTable(
396          new HTableDescriptor(tableName)
397              .addFamily(new HColumnDescriptor(foo))
398              // add the coprocessor for the region
399              .addCoprocessor(CustomRegionObserver.class.getName()));
400      try (Table table = connection.getTable(tableName)) {
401        table.get(new Get(foo));
402        table.get(new Get(foo)); // 2 gets
403      }
404    }
405
406    assertPreGetRequestsCounter(CustomRegionObserver.class);
407  }
408
409  @Test
410  public void testRegionObserverMultiRegion() throws IOException {
411    final TableName tableName = TableName.valueOf(name.getMethodName());
412    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
413         Admin admin = connection.getAdmin()) {
414      admin.createTable(
415          new HTableDescriptor(tableName)
416              .addFamily(new HColumnDescriptor(foo))
417              // add the coprocessor for the region
418              .addCoprocessor(CustomRegionObserver.class.getName())
419          , new byte[][]{foo}); // create with 2 regions
420      try (Table table = connection.getTable(tableName);
421           RegionLocator locator = connection.getRegionLocator(tableName)) {
422        table.get(new Get(bar));
423        table.get(new Get(foo)); // 2 gets to 2 separate regions
424        assertEquals(2, locator.getAllRegionLocations().size());
425        assertNotEquals(locator.getRegionLocation(bar).getRegionInfo(),
426            locator.getRegionLocation(foo).getRegionInfo());
427      }
428    }
429
430    assertPreGetRequestsCounter(CustomRegionObserver.class);
431  }
432
433  @Test
434  public void testRegionObserverMultiTable() throws IOException {
435    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
436    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
437    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
438         Admin admin = connection.getAdmin()) {
439      admin.createTable(
440          new HTableDescriptor(tableName1)
441              .addFamily(new HColumnDescriptor(foo))
442              // add the coprocessor for the region
443              .addCoprocessor(CustomRegionObserver.class.getName()));
444      admin.createTable(
445          new HTableDescriptor(tableName2)
446              .addFamily(new HColumnDescriptor(foo))
447              // add the coprocessor for the region
448              .addCoprocessor(CustomRegionObserver.class.getName()));
449      try (Table table1 = connection.getTable(tableName1);
450           Table table2 = connection.getTable(tableName2)) {
451        table1.get(new Get(bar));
452        table2.get(new Get(foo)); // 2 gets to 2 separate tables
453      }
454    }
455    assertPreGetRequestsCounter(CustomRegionObserver.class);
456  }
457
458  @Test
459  public void testRegionObserverMultiCoprocessor() throws IOException {
460    final TableName tableName = TableName.valueOf(name.getMethodName());
461    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
462         Admin admin = connection.getAdmin()) {
463      admin.createTable(
464          new HTableDescriptor(tableName)
465              .addFamily(new HColumnDescriptor(foo))
466              // add the coprocessor for the region. We add two different coprocessors
467              .addCoprocessor(CustomRegionObserver.class.getName())
468              .addCoprocessor(CustomRegionObserver2.class.getName()));
469      try (Table table = connection.getTable(tableName)) {
470        table.get(new Get(foo));
471        table.get(new Get(foo)); // 2 gets
472      }
473    }
474
475    // we will have two counters coming from two coprocs, in two different MetricRegistries
476    assertPreGetRequestsCounter(CustomRegionObserver.class);
477    assertPreGetRequestsCounter(CustomRegionObserver2.class);
478  }
479
480  @Test
481  public void testRegionObserverAfterRegionClosed() throws IOException {
482    final TableName tableName = TableName.valueOf(name.getMethodName());
483    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
484         Admin admin = connection.getAdmin()) {
485      admin.createTable(
486          new HTableDescriptor(tableName)
487              .addFamily(new HColumnDescriptor(foo))
488              // add the coprocessor for the region
489              .addCoprocessor(CustomRegionObserver.class.getName())
490          , new byte[][]{foo}); // create with 2 regions
491      try (Table table = connection.getTable(tableName)) {
492        table.get(new Get(foo));
493        table.get(new Get(foo)); // 2 gets
494      }
495
496      assertPreGetRequestsCounter(CustomRegionObserver.class);
497
498      // close one of the regions
499      try (RegionLocator locator = connection.getRegionLocator(tableName)) {
500        HRegionLocation loc = locator.getRegionLocation(foo);
501        admin.unassign(loc.getRegionInfo().getEncodedNameAsBytes(), true);
502
503        HRegionServer server = UTIL.getMiniHBaseCluster().getRegionServer(loc.getServerName());
504        UTIL.waitFor(30000,
505            () -> server.getOnlineRegion(loc.getRegionInfo().getRegionName()) == null);
506        assertNull(server.getOnlineRegion(loc.getRegionInfo().getRegionName()));
507      }
508
509      // with only 1 region remaining, we should still be able to find the Counter
510      assertPreGetRequestsCounter(CustomRegionObserver.class);
511
512      // close the table
513      admin.disableTable(tableName);
514
515      MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRegionCoprocessor(
516          CustomRegionObserver.class.getName());
517
518      // ensure that MetricRegistry is deleted
519      Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
520      assertFalse(registry.isPresent());
521    }
522  }
523
524  @Test
525  public void testRegionObserverEndpoint() throws IOException, ServiceException {
526    final TableName tableName = TableName.valueOf(name.getMethodName());
527    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
528         Admin admin = connection.getAdmin()) {
529      admin.createTable(
530          new HTableDescriptor(tableName)
531              .addFamily(new HColumnDescriptor(foo))
532              // add the coprocessor for the region
533              .addCoprocessor(CustomRegionEndpoint.class.getName()));
534
535      try (Table table = connection.getTable(tableName)) {
536        List<Mutation> mutations = Lists.newArrayList(new Put(foo), new Put(bar));
537        MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
538
539        for (Mutation mutation : mutations) {
540          mrmBuilder.addMutationRequest(ProtobufUtil.toMutation(
541              ClientProtos.MutationProto.MutationType.PUT, mutation));
542        }
543
544        CoprocessorRpcChannel channel = table.coprocessorService(bar);
545        MultiRowMutationService.BlockingInterface service =
546            MultiRowMutationService.newBlockingStub(channel);
547        MutateRowsRequest mrm = mrmBuilder.build();
548        service.mutateRows(null, mrm);
549      }
550    }
551
552    // Find out the MetricRegistry used by the CP using the global registries
553    MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRegionCoprocessor(
554        CustomRegionEndpoint.class.getName());
555
556    Optional<MetricRegistry> registry =  MetricRegistries.global().get(info);
557    assertTrue(registry.isPresent());
558
559    Optional<Metric> metric = registry.get().get("EndpointExecution");
560    assertTrue(metric.isPresent());
561
562    Timer endpointExecutions = (Timer)metric.get();
563    assertEquals(1, endpointExecutions.getHistogram().getCount());
564  }
565}