001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static junit.framework.TestCase.assertEquals;
021
022import java.util.ArrayList;
023import java.util.List;
024import java.util.concurrent.ThreadLocalRandom;
025import org.apache.hadoop.hbase.Cell;
026import org.apache.hadoop.hbase.CellBuilderFactory;
027import org.apache.hadoop.hbase.CellBuilderType;
028import org.apache.hadoop.hbase.CompatibilityFactory;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HColumnDescriptor;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.HTableDescriptor;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.Waiter;
036import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
037import org.apache.hadoop.hbase.ipc.RpcServerInterface;
038import org.apache.hadoop.hbase.metrics.BaseSource;
039import org.apache.hadoop.hbase.regionserver.HRegionServer;
040import org.apache.hadoop.hbase.test.MetricsAssertHelper;
041import org.apache.hadoop.hbase.testclassification.ClientTests;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.junit.AfterClass;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Rule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050import org.junit.rules.TestName;
051
052/**
053 * This test sets the multi size WAAAAAY low and then checks to make sure that gets will still make
054 * progress.
055 */
056@Category({MediumTests.class, ClientTests.class})
057public class TestMultiRespectsLimits {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061      HBaseClassTestRule.forClass(TestMultiRespectsLimits.class);
062
063  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
064  private static final MetricsAssertHelper METRICS_ASSERT =
065      CompatibilityFactory.getInstance(MetricsAssertHelper.class);
066  private final static byte[] FAMILY = Bytes.toBytes("D");
067  public static final int MAX_SIZE = 500;
068
069  @Rule
070  public TestName name = new TestName();
071
072  @BeforeClass
073  public static void setUpBeforeClass() throws Exception {
074    TEST_UTIL.getConfiguration().setLong(
075        HConstants.HBASE_SERVER_SCANNER_MAX_RESULT_SIZE_KEY,
076        MAX_SIZE);
077
078    // Only start on regionserver so that all regions are on the same server.
079    TEST_UTIL.startMiniCluster(1);
080  }
081
082  @AfterClass
083  public static void tearDownAfterClass() throws Exception {
084    TEST_UTIL.shutdownMiniCluster();
085  }
086
087  @Test
088  public void testMultiLimits() throws Exception {
089    final TableName tableName = TableName.valueOf(name.getMethodName());
090    Table t = TEST_UTIL.createTable(tableName, FAMILY);
091    TEST_UTIL.loadTable(t, FAMILY, false);
092
093    // Split the table to make sure that the chunking happens accross regions.
094    try (final Admin admin = TEST_UTIL.getAdmin()) {
095      admin.split(tableName);
096      TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
097        @Override
098        public boolean evaluate() throws Exception {
099          return admin.getTableRegions(tableName).size() > 1;
100        }
101      });
102    }
103    List<Get> gets = new ArrayList<>(MAX_SIZE);
104
105    for (int i = 0; i < MAX_SIZE; i++) {
106      gets.add(new Get(HBaseTestingUtility.ROWS[i]));
107    }
108
109    RpcServerInterface rpcServer = TEST_UTIL.getHBaseCluster().getRegionServer(0).getRpcServer();
110    BaseSource s = rpcServer.getMetrics().getMetricsSource();
111    long startingExceptions = METRICS_ASSERT.getCounter("exceptions", s);
112    long startingMultiExceptions = METRICS_ASSERT.getCounter("exceptions.multiResponseTooLarge", s);
113
114    Result[] results = t.get(gets);
115    assertEquals(MAX_SIZE, results.length);
116
117    // Cells from TEST_UTIL.loadTable have a length of 27.
118    // Multiplying by less than that gives an easy lower bound on size.
119    // However in reality each kv is being reported as much higher than that.
120    METRICS_ASSERT.assertCounterGt("exceptions",
121        startingExceptions + ((MAX_SIZE * 25) / MAX_SIZE), s);
122    METRICS_ASSERT.assertCounterGt("exceptions.multiResponseTooLarge",
123        startingMultiExceptions + ((MAX_SIZE * 25) / MAX_SIZE), s);
124  }
125
126  @Test
127  public void testBlockMultiLimits() throws Exception {
128    final TableName tableName = TableName.valueOf(name.getMethodName());
129    HTableDescriptor desc = new HTableDescriptor(tableName);
130    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
131    hcd.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
132    desc.addFamily(hcd);
133    TEST_UTIL.getAdmin().createTable(desc);
134    Table t = TEST_UTIL.getConnection().getTable(tableName);
135
136    final HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0);
137    RpcServerInterface rpcServer = regionServer.getRpcServer();
138    BaseSource s = rpcServer.getMetrics().getMetricsSource();
139    long startingExceptions = METRICS_ASSERT.getCounter("exceptions", s);
140    long startingMultiExceptions = METRICS_ASSERT.getCounter("exceptions.multiResponseTooLarge", s);
141
142    byte[] row = Bytes.toBytes("TEST");
143    byte[][] cols = new byte[][]{
144        Bytes.toBytes("0"), // Get this
145        Bytes.toBytes("1"), // Buffer
146        Bytes.toBytes("2"), // Buffer
147        Bytes.toBytes("3"), // Get This
148        Bytes.toBytes("4"), // Buffer
149        Bytes.toBytes("5"), // Buffer
150    };
151
152    // Set the value size so that one result will be less than the MAX_SIE
153    // however the block being reference will be larger than MAX_SIZE.
154    // This should cause the regionserver to try and send a result immediately.
155    byte[] value = new byte[MAX_SIZE - 100];
156    ThreadLocalRandom.current().nextBytes(value);
157
158    for (byte[] col:cols) {
159      Put p = new Put(row);
160      p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
161              .setRow(row)
162              .setFamily(FAMILY)
163              .setQualifier(col)
164              .setTimestamp(p.getTimestamp())
165              .setType(Cell.Type.Put)
166              .setValue(value)
167              .build());
168      t.put(p);
169    }
170
171    // Make sure that a flush happens
172    try (final Admin admin = TEST_UTIL.getAdmin()) {
173      admin.flush(tableName);
174      TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
175        @Override
176        public boolean evaluate() throws Exception {
177          return regionServer.getRegions(tableName).get(0).getMaxFlushedSeqId() > 3;
178        }
179      });
180    }
181
182    List<Get> gets = new ArrayList<>(2);
183    Get g0 = new Get(row);
184    g0.addColumn(FAMILY, cols[0]);
185    gets.add(g0);
186
187    Get g2 = new Get(row);
188    g2.addColumn(FAMILY, cols[3]);
189    gets.add(g2);
190
191    Result[] results = t.get(gets);
192    assertEquals(2, results.length);
193    METRICS_ASSERT.assertCounterGt("exceptions", startingExceptions, s);
194    METRICS_ASSERT.assertCounterGt("exceptions.multiResponseTooLarge",
195        startingMultiExceptions, s);
196  }
197}