Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.List;
028import java.util.Optional;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.CompatibilityFactory;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.HRegionLocation;
035import org.apache.hadoop.hbase.NamespaceDescriptor;
036import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Admin;
039import org.apache.hadoop.hbase.client.Append;
040import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
042import org.apache.hadoop.hbase.client.Connection;
043import org.apache.hadoop.hbase.client.Durability;
044import org.apache.hadoop.hbase.client.Get;
045import org.apache.hadoop.hbase.client.Increment;
046import org.apache.hadoop.hbase.client.Put;
047import org.apache.hadoop.hbase.client.RegionInfo;
048import org.apache.hadoop.hbase.client.RegionLocator;
049import org.apache.hadoop.hbase.client.Result;
050import org.apache.hadoop.hbase.client.ResultScanner;
051import org.apache.hadoop.hbase.client.Scan;
052import org.apache.hadoop.hbase.client.Scan.ReadType;
053import org.apache.hadoop.hbase.client.Table;
054import org.apache.hadoop.hbase.client.TableDescriptor;
055import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
056import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
057import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
058import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
059import org.apache.hadoop.hbase.security.User;
060import org.apache.hadoop.hbase.test.MetricsAssertHelper;
061import org.apache.hadoop.hbase.testclassification.LargeTests;
062import org.apache.hadoop.hbase.testclassification.RegionServerTests;
063import org.apache.hadoop.hbase.util.Bytes;
064import org.apache.hadoop.hbase.util.Threads;
065import org.junit.After;
066import org.junit.AfterClass;
067import org.junit.Before;
068import org.junit.BeforeClass;
069import org.junit.ClassRule;
070import org.junit.Ignore;
071import org.junit.Rule;
072import org.junit.Test;
073import org.junit.experimental.categories.Category;
074import org.junit.rules.TestName;
075import org.slf4j.Logger;
076import org.slf4j.LoggerFactory;
077
078@Category({ RegionServerTests.class, LargeTests.class })
079public class TestRegionServerMetrics {
080
081  @ClassRule
082  public static final HBaseClassTestRule CLASS_RULE =
083    HBaseClassTestRule.forClass(TestRegionServerMetrics.class);
084
085  private static final Logger LOG = LoggerFactory.getLogger(TestRegionServerMetrics.class);
086
087  @Rule
088  public TestName testName = new TestName();
089
090  private static MetricsAssertHelper metricsHelper;
091  private static SingleProcessHBaseCluster cluster;
092  private static HRegionServer rs;
093  private static Configuration conf;
094  private static HBaseTestingUtil TEST_UTIL;
095  private static Connection connection;
096  private static MetricsRegionServer metricsRegionServer;
097  private static MetricsRegionServerSource serverSource;
098  private static final int NUM_SCAN_NEXT = 30;
099  private static int numScanNext = 0;
100  private static byte[] cf = Bytes.toBytes("cf");
101  private static byte[] row = Bytes.toBytes("row");
102  private static byte[] qualifier = Bytes.toBytes("qual");
103  private static byte[] val = Bytes.toBytes("val");
104  private static Admin admin;
105
106  @BeforeClass
107  public static void startCluster() throws Exception {
108    metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
109    TEST_UTIL = new HBaseTestingUtil();
110    conf = TEST_UTIL.getConfiguration();
111    conf.getLong("hbase.splitlog.max.resubmit", 0);
112    // Make the failure test faster
113    conf.setInt("zookeeper.recovery.retry", 0);
114    // testMobMetrics creates few hfiles and manages compaction manually.
115    conf.setInt("hbase.hstore.compactionThreshold", 100);
116    conf.setInt("hbase.hstore.compaction.max", 100);
117    conf.setInt("hbase.regionserver.periodicmemstoreflusher.rangeofdelayseconds", 4 * 60);
118    conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
119
120    TEST_UTIL.startMiniCluster();
121    cluster = TEST_UTIL.getHBaseCluster();
122    cluster.waitForActiveAndReadyMaster();
123    admin = TEST_UTIL.getAdmin();
124    connection = TEST_UTIL.getConnection();
125
126    while (
127      cluster.getLiveRegionServerThreads().isEmpty() && cluster.getRegionServer(0) == null
128        && rs.getMetrics() == null
129    ) {
130      Threads.sleep(100);
131    }
132    rs = cluster.getRegionServer(0);
133    metricsRegionServer = rs.getMetrics();
134    serverSource = metricsRegionServer.getMetricsSource();
135  }
136
137  @AfterClass
138  public static void after() throws Exception {
139    if (TEST_UTIL != null) {
140      TEST_UTIL.shutdownMiniCluster();
141    }
142  }
143
144  TableName tableName;
145  Table table;
146
147  @Before
148  public void beforeTestMethod() throws Exception {
149    metricsRegionServer.getRegionServerWrapper().forceRecompute();
150    tableName = TableName.valueOf(testName.getMethodName());
151    table = TEST_UTIL.createTable(tableName, cf);
152  }
153
154  @After
155  public void afterTestMethod() throws Exception {
156    admin.disableTable(tableName);
157    admin.deleteTable(tableName);
158  }
159
160  private void assertCounter(String metric, long expectedValue) {
161    metricsHelper.assertCounter(metric, expectedValue, serverSource);
162  }
163
164  private void assertGauge(String metric, long expectedValue) {
165    metricsHelper.assertGauge(metric, expectedValue, serverSource);
166  }
167
168  // Aggregates metrics from regions and assert given list of metrics and expected values.
169  private void assertRegionMetrics(String metric, long expectedValue) throws Exception {
170    try (RegionLocator locator = connection.getRegionLocator(tableName)) {
171      for (HRegionLocation location : locator.getAllRegionLocations()) {
172        RegionInfo hri = location.getRegion();
173        MetricsRegionAggregateSource agg =
174          rs.getRegion(hri.getRegionName()).getMetrics().getSource().getAggregateSource();
175        String prefix = "namespace_" + NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR + "_table_"
176          + tableName.getNameAsString() + "_region_" + hri.getEncodedName() + "_metric_";
177        metricsHelper.assertCounter(prefix + metric, expectedValue, agg);
178      }
179    }
180  }
181
182  private void doNPuts(int n, boolean batch) throws Exception {
183    if (batch) {
184      List<Put> puts = new ArrayList<>();
185      for (int i = 0; i < n; i++) {
186        Put p = new Put(Bytes.toBytes("" + i + "row")).addColumn(cf, qualifier, val);
187        puts.add(p);
188      }
189      table.put(puts);
190    } else {
191      for (int i = 0; i < n; i++) {
192        Put p = new Put(row).addColumn(cf, qualifier, val);
193        table.put(p);
194      }
195    }
196  }
197
198  private void doNGets(int n, boolean batch) throws Exception {
199    if (batch) {
200      List<Get> gets = new ArrayList<>();
201      for (int i = 0; i < n; i++) {
202        gets.add(new Get(row));
203      }
204      table.get(gets);
205    } else {
206      for (int i = 0; i < n; i++) {
207        table.get(new Get(row));
208      }
209    }
210  }
211
212  private void doScan(int n, boolean caching) throws IOException {
213    Scan scan = new Scan();
214    if (caching) {
215      scan.setCaching(n);
216    } else {
217      scan.setCaching(1);
218    }
219    ResultScanner scanner = table.getScanner(scan);
220    for (int i = 0; i < n; i++) {
221      Result res = scanner.next();
222      LOG.debug("Result row: " + Bytes.toString(res.getRow()) + ", value: "
223        + Bytes.toString(res.getValue(cf, qualifier)));
224    }
225  }
226
227  @Test
228  public void testRegionCount() throws Exception {
229    metricsHelper.assertGauge("regionCount", 2, serverSource);
230  }
231
232  @Test
233  public void testLocalFiles() throws Exception {
234    assertGauge("percentFilesLocal", 0);
235    assertGauge("percentFilesLocalSecondaryRegions", 0);
236  }
237
238  @Test
239  public void testRequestCount() throws Exception {
240    // Do a first put to be sure that the connection is established, meta is there and so on.
241    doNPuts(1, false);
242
243    metricsRegionServer.getRegionServerWrapper().forceRecompute();
244    long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
245    long rowActionRequests = metricsHelper.getCounter("totalRowActionRequestCount", serverSource);
246    long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
247    long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
248
249    doNPuts(30, false);
250
251    metricsRegionServer.getRegionServerWrapper().forceRecompute();
252    assertCounter("totalRequestCount", requests + 30);
253    assertCounter("totalRowActionRequestCount", rowActionRequests + 30);
254    assertCounter("readRequestCount", readRequests);
255    assertCounter("writeRequestCount", writeRequests + 30);
256
257    doNGets(10, false);
258
259    metricsRegionServer.getRegionServerWrapper().forceRecompute();
260    assertCounter("totalRequestCount", requests + 40);
261    assertCounter("totalRowActionRequestCount", rowActionRequests + 40);
262    assertCounter("readRequestCount", readRequests + 10);
263    assertCounter("writeRequestCount", writeRequests + 30);
264
265    assertRegionMetrics("getCount", 10);
266    assertRegionMetrics("putCount", 31);
267
268    doNGets(10, true); // true = batch
269
270    metricsRegionServer.getRegionServerWrapper().forceRecompute();
271
272    assertCounter("writeRequestCount", writeRequests + 30);
273
274    doNPuts(30, true);
275
276    metricsRegionServer.getRegionServerWrapper().forceRecompute();
277    assertCounter("writeRequestCount", writeRequests + 60);
278
279    doScan(10, false); // test after batch put so we have enough lines
280    metricsRegionServer.getRegionServerWrapper().forceRecompute();
281    assertCounter("writeRequestCount", writeRequests + 60);
282    numScanNext += 10;
283
284    doScan(10, true); // true = caching
285    metricsRegionServer.getRegionServerWrapper().forceRecompute();
286    assertCounter("writeRequestCount", writeRequests + 60);
287    numScanNext += 1;
288  }
289
290  @Test
291  public void testGet() throws Exception {
292    // Do a first put to be sure that the connection is established, meta is there and so on.
293    doNPuts(1, false);
294    doNGets(10, false);
295    assertRegionMetrics("getCount", 10);
296    metricsHelper.assertCounterGt("Get_num_ops", 10, serverSource);
297  }
298
299  @Test
300  public void testMutationsWithoutWal() throws Exception {
301    Put p = new Put(row).addColumn(cf, qualifier, val).setDurability(Durability.SKIP_WAL);
302    table.put(p);
303
304    metricsRegionServer.getRegionServerWrapper().forceRecompute();
305    assertGauge("mutationsWithoutWALCount", 1);
306    long minLength = row.length + cf.length + qualifier.length + val.length;
307    metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
308  }
309
310  @Test
311  public void testStoreCount() throws Exception {
312    // Force a hfile.
313    doNPuts(1, false);
314    TEST_UTIL.getAdmin().flush(tableName);
315
316    metricsRegionServer.getRegionServerWrapper().forceRecompute();
317    assertGauge("storeCount", 5);
318    assertGauge("storeFileCount", 1);
319  }
320
321  @Test
322  public void testStoreFileAge() throws Exception {
323    // Force a hfile.
324    doNPuts(1, false);
325    TEST_UTIL.getAdmin().flush(tableName);
326
327    metricsRegionServer.getRegionServerWrapper().forceRecompute();
328    assertTrue(metricsHelper.getGaugeLong("maxStoreFileAge", serverSource) > 0);
329    assertTrue(metricsHelper.getGaugeLong("minStoreFileAge", serverSource) > 0);
330    assertTrue(metricsHelper.getGaugeLong("avgStoreFileAge", serverSource) > 0);
331  }
332
333  @Test
334  public void testCheckAndPutCount() throws Exception {
335    byte[] valOne = Bytes.toBytes("Value");
336    byte[] valTwo = Bytes.toBytes("ValueTwo");
337    byte[] valThree = Bytes.toBytes("ValueThree");
338
339    Put p = new Put(row);
340    p.addColumn(cf, qualifier, valOne);
341    table.put(p);
342
343    Put pTwo = new Put(row);
344    pTwo.addColumn(cf, qualifier, valTwo);
345    table.checkAndMutate(row, cf).qualifier(qualifier).ifEquals(valOne).thenPut(pTwo);
346
347    Put pThree = new Put(row);
348    pThree.addColumn(cf, qualifier, valThree);
349    table.checkAndMutate(row, cf).qualifier(qualifier).ifEquals(valOne).thenPut(pThree);
350
351    metricsRegionServer.getRegionServerWrapper().forceRecompute();
352    assertCounter("checkMutateFailedCount", 1);
353    assertCounter("checkMutatePassedCount", 1);
354  }
355
356  @Test
357  public void testIncrement() throws Exception {
358    Put p = new Put(row).addColumn(cf, qualifier, Bytes.toBytes(0L));
359    table.put(p);
360
361    for (int count = 0; count < 13; count++) {
362      Increment inc = new Increment(row);
363      inc.addColumn(cf, qualifier, 100);
364      table.increment(inc);
365    }
366
367    metricsRegionServer.getRegionServerWrapper().forceRecompute();
368    assertCounter("incrementNumOps", 13);
369  }
370
371  @Test
372  public void testAppend() throws Exception {
373    doNPuts(1, false);
374
375    for (int count = 0; count < 73; count++) {
376      Append append = new Append(row);
377      append.addColumn(cf, qualifier, Bytes.toBytes(",Test"));
378      table.append(append);
379    }
380
381    metricsRegionServer.getRegionServerWrapper().forceRecompute();
382    assertCounter("appendNumOps", 73);
383  }
384
385  @Test
386  public void testScanSize() throws Exception {
387    doNPuts(100, true); // batch put
388    Scan s = new Scan();
389    s.setBatch(1).setCaching(1).setLimit(NUM_SCAN_NEXT).setReadType(ReadType.STREAM);
390    try (ResultScanner resultScanners = table.getScanner(s)) {
391      for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
392        Result result = resultScanners.next();
393        assertNotNull(result);
394        assertEquals(1, result.size());
395      }
396      numScanNext += NUM_SCAN_NEXT;
397      assertRegionMetrics("scanCount", NUM_SCAN_NEXT);
398    }
399  }
400
401  @Test
402  public void testScanTime() throws Exception {
403    doNPuts(100, true);
404    Scan s = new Scan();
405    s.setBatch(1).setCaching(1).setLimit(NUM_SCAN_NEXT);
406    try (ResultScanner resultScanners = table.getScanner(s)) {
407      for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
408        Result result = resultScanners.next();
409        assertNotNull(result);
410        assertEquals(1, result.size());
411      }
412    }
413    numScanNext += NUM_SCAN_NEXT;
414    assertRegionMetrics("scanCount", NUM_SCAN_NEXT);
415  }
416
417  @Test
418  public void testScanSizeForSmallScan() throws Exception {
419    doNPuts(100, true);
420    Scan s = new Scan();
421    s.setCaching(1).setLimit(NUM_SCAN_NEXT).setReadType(ReadType.PREAD);
422    try (ResultScanner resultScanners = table.getScanner(s)) {
423      for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
424        Result result = resultScanners.next();
425        assertNotNull(result);
426      }
427      assertNull(resultScanners.next());
428    }
429    numScanNext += NUM_SCAN_NEXT;
430    assertRegionMetrics("scanCount", NUM_SCAN_NEXT);
431  }
432
433  @Test
434  public void testMobMetrics() throws IOException, InterruptedException {
435    TableName tableName = TableName.valueOf("testMobMetricsLocal");
436    int numHfiles = 5;
437    TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
438      .setColumnFamily(
439        ColumnFamilyDescriptorBuilder.newBuilder(cf).setMobEnabled(true).setMobThreshold(0).build())
440      .build();
441    byte[] val = Bytes.toBytes("mobdata");
442    try {
443      Table table = TEST_UTIL.createTable(htd, new byte[0][0], conf);
444      HRegion region = rs.getRegions(tableName).get(0);
445      for (int insertCount = 0; insertCount < numHfiles; insertCount++) {
446        Put p = new Put(Bytes.toBytes(insertCount));
447        p.addColumn(cf, qualifier, val);
448        table.put(p);
449        admin.flush(tableName);
450      }
451      metricsRegionServer.getRegionServerWrapper().forceRecompute();
452      assertCounter("mobFlushCount", numHfiles);
453      Scan scan = new Scan().withStartRow(Bytes.toBytes(0)).withStopRow(Bytes.toBytes(numHfiles));
454      ResultScanner scanner = table.getScanner(scan);
455      scanner.next(100);
456      numScanNext++; // this is an ugly construct
457      scanner.close();
458      metricsRegionServer.getRegionServerWrapper().forceRecompute();
459      assertCounter("mobScanCellsCount", numHfiles);
460
461      setMobThreshold(region, cf, 100);
462      // metrics are reset by the region initialization
463      region.initialize();
464      // This is how we MOB compact region
465      List<HStore> stores = region.getStores();
466      for (HStore store : stores) {
467        // Force major compaction
468        store.triggerMajorCompaction();
469        Optional<CompactionContext> context = store.requestCompaction(HStore.PRIORITY_USER,
470          CompactionLifeCycleTracker.DUMMY, User.getCurrent());
471        if (!context.isPresent()) {
472          continue;
473        }
474        region.compact(context.get(), store, NoLimitThroughputController.INSTANCE,
475          User.getCurrent());
476      }
477      metricsRegionServer.getRegionServerWrapper().forceRecompute();
478      assertCounter("cellsCountCompactedFromMob", numHfiles);
479      assertCounter("cellsCountCompactedToMob", 0);
480
481      scanner = table.getScanner(scan);
482      scanner.next(100);
483      numScanNext++; // this is an ugly construct
484      metricsRegionServer.getRegionServerWrapper().forceRecompute();
485      assertCounter("mobScanCellsCount", 0);
486
487      for (int insertCount = numHfiles; insertCount < 2 * numHfiles; insertCount++) {
488        Put p = new Put(Bytes.toBytes(insertCount));
489        p.addColumn(cf, qualifier, val);
490        table.put(p);
491        admin.flush(tableName);
492      }
493      setMobThreshold(region, cf, 0);
494
495      // closing the region forces the compaction.discharger to archive the compacted hfiles
496      region.close();
497
498      // metrics are reset by the region initialization
499      region.initialize();
500      region.compact(true);
501      metricsRegionServer.getRegionServerWrapper().forceRecompute();
502      // metrics are reset by the region initialization
503      assertCounter("cellsCountCompactedFromMob", 0);
504      assertCounter("cellsCountCompactedToMob", 2 * numHfiles);
505    } finally {
506      admin.disableTable(tableName);
507      admin.deleteTable(tableName);
508    }
509  }
510
511  private static Region setMobThreshold(Region region, byte[] cfName, long modThreshold) {
512    ColumnFamilyDescriptor cfd =
513      ColumnFamilyDescriptorBuilder.newBuilder(region.getTableDescriptor().getColumnFamily(cfName))
514        .setMobThreshold(modThreshold).build();
515    TableDescriptor td = TableDescriptorBuilder.newBuilder(region.getTableDescriptor())
516      .removeColumnFamily(cfName).setColumnFamily(cfd).build();
517    ((HRegion) region).setTableDescriptor(td);
518    return region;
519  }
520
521  @Test
522  @Ignore
523  public void testRangeCountMetrics() throws Exception {
524    final long[] timeranges =
525      { 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 60000, 120000, 300000, 600000 };
526    final String timeRangeType = "TimeRangeCount";
527    final String timeRangeMetricName = "Mutate";
528    boolean timeRangeCountUpdated = false;
529
530    // Do a first put to be sure that the connection is established, meta is there and so on.
531    Put p = new Put(row);
532    p.addColumn(cf, qualifier, val);
533    table.put(p);
534
535    // do some puts and gets
536    for (int i = 0; i < 10; i++) {
537      table.put(p);
538    }
539
540    Get g = new Get(row);
541    for (int i = 0; i < 10; i++) {
542      table.get(g);
543    }
544
545    metricsRegionServer.getRegionServerWrapper().forceRecompute();
546
547    // Check some time range counters were updated
548    long prior = 0;
549
550    String dynamicMetricName;
551    for (int i = 0; i < timeranges.length; i++) {
552      dynamicMetricName =
553        timeRangeMetricName + "_" + timeRangeType + "_" + prior + "-" + timeranges[i];
554      if (metricsHelper.checkCounterExists(dynamicMetricName, serverSource)) {
555        long count = metricsHelper.getGaugeLong(dynamicMetricName, serverSource);
556        if (count > 0) {
557          timeRangeCountUpdated = true;
558          break;
559        }
560      }
561      prior = timeranges[i];
562    }
563    dynamicMetricName =
564      timeRangeMetricName + "_" + timeRangeType + "_" + timeranges[timeranges.length - 1] + "-inf";
565    if (metricsHelper.checkCounterExists(dynamicMetricName, serverSource)) {
566      long count = metricsHelper.getCounter(dynamicMetricName, serverSource);
567      if (count > 0) {
568        timeRangeCountUpdated = true;
569      }
570    }
571    assertEquals(true, timeRangeCountUpdated);
572  }
573
574  @Test
575  public void testAverageRegionSize() throws Exception {
576    // Force a hfile.
577    doNPuts(1, false);
578    TEST_UTIL.getAdmin().flush(tableName);
579
580    metricsRegionServer.getRegionServerWrapper().forceRecompute();
581    assertTrue(metricsHelper.getGaugeDouble("averageRegionSize", serverSource) > 0.0);
582  }
583
584  @Test
585  public void testReadBytes() throws Exception {
586    // Do a first put to be sure that the connection is established, meta is there and so on.
587    doNPuts(1, false);
588    doNGets(10, false);
589    TEST_UTIL.getAdmin().flush(tableName);
590    metricsRegionServer.getRegionServerWrapper().forceRecompute();
591
592    assertTrue("Total read bytes should be larger than 0",
593      metricsRegionServer.getRegionServerWrapper().getTotalBytesRead() > 0);
594    assertTrue("Total local read bytes should be larger than 0",
595      metricsRegionServer.getRegionServerWrapper().getLocalBytesRead() > 0);
596    assertEquals("Total short circuit read bytes should be equal to 0", 0,
597      metricsRegionServer.getRegionServerWrapper().getShortCircuitBytesRead());
598    assertEquals("Total zero-byte read bytes should be equal to 0", 0,
599      metricsRegionServer.getRegionServerWrapper().getZeroCopyBytesRead());
600  }
601}