001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021
022import java.util.ArrayList;
023import java.util.List;
024import java.util.concurrent.ThreadLocalRandom;
025import org.apache.hadoop.hbase.Cell;
026import org.apache.hadoop.hbase.CellBuilderFactory;
027import org.apache.hadoop.hbase.CellBuilderType;
028import org.apache.hadoop.hbase.CompatibilityFactory;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.Waiter;
034import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
035import org.apache.hadoop.hbase.ipc.RpcServerInterface;
036import org.apache.hadoop.hbase.metrics.BaseSource;
037import org.apache.hadoop.hbase.regionserver.HRegionServer;
038import org.apache.hadoop.hbase.test.MetricsAssertHelper;
039import org.apache.hadoop.hbase.testclassification.ClientTests;
040import org.apache.hadoop.hbase.testclassification.MediumTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.log4j.Level;
043import org.apache.log4j.LogManager;
044import org.junit.AfterClass;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Rule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050import org.junit.rules.TestName;
051
052/**
053 * This test sets the multi size WAAAAAY low and then checks to make sure that gets will still make
054 * progress.
055 */
056@Category({MediumTests.class, ClientTests.class})
057public class TestMultiRespectsLimits {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061      HBaseClassTestRule.forClass(TestMultiRespectsLimits.class);
062
063  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
064  private static final MetricsAssertHelper METRICS_ASSERT =
065      CompatibilityFactory.getInstance(MetricsAssertHelper.class);
066  private final static byte[] FAMILY = Bytes.toBytes("D");
067  public static final int MAX_SIZE = 100;
068
069  @Rule
070  public TestName name = new TestName();
071
072  @BeforeClass
073  public static void setUpBeforeClass() throws Exception {
074    // disable the debug log to avoid flooding the output
075    LogManager.getLogger(AsyncRegionLocatorHelper.class).setLevel(Level.INFO);
076    TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_SERVER_SCANNER_MAX_RESULT_SIZE_KEY,
077      MAX_SIZE);
078
079    // Only start on regionserver so that all regions are on the same server.
080    TEST_UTIL.startMiniCluster(1);
081  }
082
083  @AfterClass
084  public static void tearDownAfterClass() throws Exception {
085    TEST_UTIL.shutdownMiniCluster();
086  }
087
088  @Test
089  public void testMultiLimits() throws Exception {
090    final TableName tableName = TableName.valueOf(name.getMethodName());
091    Table t = TEST_UTIL.createTable(tableName, FAMILY);
092    TEST_UTIL.loadTable(t, FAMILY, false);
093
094    // Split the table to make sure that the chunking happens accross regions.
095    try (final Admin admin = TEST_UTIL.getAdmin()) {
096      admin.split(tableName);
097      TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
098        @Override
099        public boolean evaluate() throws Exception {
100          return admin.getRegions(tableName).size() > 1;
101        }
102      });
103    }
104    List<Get> gets = new ArrayList<>(MAX_SIZE);
105
106    for (int i = 0; i < MAX_SIZE; i++) {
107      gets.add(new Get(HBaseTestingUtility.ROWS[i]));
108    }
109
110    RpcServerInterface rpcServer = TEST_UTIL.getHBaseCluster().getRegionServer(0).getRpcServer();
111    BaseSource s = rpcServer.getMetrics().getMetricsSource();
112    long startingExceptions = METRICS_ASSERT.getCounter("exceptions", s);
113    long startingMultiExceptions = METRICS_ASSERT.getCounter("exceptions.multiResponseTooLarge", s);
114
115    Result[] results = t.get(gets);
116    assertEquals(MAX_SIZE, results.length);
117
118    // Cells from TEST_UTIL.loadTable have a length of 27.
119    // Multiplying by less than that gives an easy lower bound on size.
120    // However in reality each kv is being reported as much higher than that.
121    METRICS_ASSERT.assertCounterGt("exceptions",
122        startingExceptions + ((MAX_SIZE * 25) / MAX_SIZE), s);
123    METRICS_ASSERT.assertCounterGt("exceptions.multiResponseTooLarge",
124        startingMultiExceptions + ((MAX_SIZE * 25) / MAX_SIZE), s);
125  }
126
127  @Test
128  public void testBlockMultiLimits() throws Exception {
129    final TableName tableName = TableName.valueOf(name.getMethodName());
130    TEST_UTIL.getAdmin().createTable(
131      TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder
132        .newBuilder(FAMILY).setDataBlockEncoding(DataBlockEncoding.FAST_DIFF).build()).build());
133    Table t = TEST_UTIL.getConnection().getTable(tableName);
134
135    final HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0);
136    RpcServerInterface rpcServer = regionServer.getRpcServer();
137    BaseSource s = rpcServer.getMetrics().getMetricsSource();
138    long startingExceptions = METRICS_ASSERT.getCounter("exceptions", s);
139    long startingMultiExceptions = METRICS_ASSERT.getCounter("exceptions.multiResponseTooLarge", s);
140
141    byte[] row = Bytes.toBytes("TEST");
142    byte[][] cols = new byte[][]{
143        Bytes.toBytes("0"), // Get this
144        Bytes.toBytes("1"), // Buffer
145        Bytes.toBytes("2"), // Buffer
146        Bytes.toBytes("3"), // Get This
147        Bytes.toBytes("4"), // Buffer
148        Bytes.toBytes("5"), // Buffer
149    };
150
151    // Set the value size so that one result will be less than the MAX_SIE
152    // however the block being reference will be larger than MAX_SIZE.
153    // This should cause the regionserver to try and send a result immediately.
154    byte[] value = new byte[MAX_SIZE - 100];
155    ThreadLocalRandom.current().nextBytes(value);
156
157    for (byte[] col:cols) {
158      Put p = new Put(row);
159      p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
160              .setRow(row)
161              .setFamily(FAMILY)
162              .setQualifier(col)
163              .setTimestamp(p.getTimestamp())
164              .setType(Cell.Type.Put)
165              .setValue(value)
166              .build());
167      t.put(p);
168    }
169
170    // Make sure that a flush happens
171    try (final Admin admin = TEST_UTIL.getAdmin()) {
172      admin.flush(tableName);
173      TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
174        @Override
175        public boolean evaluate() throws Exception {
176          return regionServer.getRegions(tableName).get(0).getMaxFlushedSeqId() > 3;
177        }
178      });
179    }
180
181    List<Get> gets = new ArrayList<>(2);
182    Get g0 = new Get(row);
183    g0.addColumn(FAMILY, cols[0]);
184    gets.add(g0);
185
186    Get g2 = new Get(row);
187    g2.addColumn(FAMILY, cols[3]);
188    gets.add(g2);
189
190    Result[] results = t.get(gets);
191    assertEquals(2, results.length);
192    METRICS_ASSERT.assertCounterGt("exceptions", startingExceptions, s);
193    METRICS_ASSERT.assertCounterGt("exceptions.multiResponseTooLarge",
194        startingMultiExceptions, s);
195  }
196}