001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021
022import java.util.ArrayList;
023import java.util.LinkedList;
024import java.util.List;
025import java.util.Map.Entry;
026import org.apache.hadoop.hbase.HBaseClassTestRule;
027import org.apache.hadoop.hbase.HBaseTestingUtil;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.client.Scan.ReadType;
030import org.apache.hadoop.hbase.testclassification.LargeTests;
031import org.apache.hadoop.hbase.util.Bytes;
032import org.junit.AfterClass;
033import org.junit.BeforeClass;
034import org.junit.ClassRule;
035import org.junit.Test;
036import org.junit.experimental.categories.Category;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
041
042@Category(LargeTests.class)
043public class TestSizeFailures {
044
045  @ClassRule
046  public static final HBaseClassTestRule CLASS_RULE =
047    HBaseClassTestRule.forClass(TestSizeFailures.class);
048
049  private static final Logger LOG = LoggerFactory.getLogger(TestSizeFailures.class);
050  protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
051  private static byte[] FAMILY = Bytes.toBytes("testFamily");
052  protected static int SLAVES = 1;
053  private static TableName TABLENAME;
054  private static final int NUM_ROWS = 1000 * 1000, NUM_COLS = 9;
055
056  @BeforeClass
057  public static void setUpBeforeClass() throws Exception {
058    // Uncomment the following lines if more verbosity is needed for
059    // debugging (see HBASE-12285 for details).
060    // ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
061    // ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
062    // ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
063    TEST_UTIL.startMiniCluster(SLAVES);
064
065    // Write a bunch of data
066    TABLENAME = TableName.valueOf("testSizeFailures");
067    List<byte[]> qualifiers = new ArrayList<>();
068    for (int i = 1; i <= 10; i++) {
069      qualifiers.add(Bytes.toBytes(Integer.toString(i)));
070    }
071
072    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TABLENAME)
073      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build();
074    byte[][] splits = new byte[9][2];
075    for (int i = 1; i < 10; i++) {
076      int split = 48 + i;
077      splits[i - 1][0] = (byte) (split >>> 8);
078      splits[i - 1][0] = (byte) (split);
079    }
080    TEST_UTIL.getAdmin().createTable(tableDescriptor, splits);
081    Connection conn = TEST_UTIL.getConnection();
082
083    try (Table table = conn.getTable(TABLENAME)) {
084      List<Put> puts = new LinkedList<>();
085      for (int i = 0; i < NUM_ROWS; i++) {
086        Put p = new Put(Bytes.toBytes(Integer.toString(i)));
087        for (int j = 0; j < NUM_COLS; j++) {
088          byte[] value = new byte[50];
089          Bytes.random(value);
090          p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value);
091        }
092        puts.add(p);
093
094        if (puts.size() == 1000) {
095          table.batch(puts, null);
096          puts.clear();
097        }
098      }
099
100      if (puts.size() > 0) {
101        table.batch(puts, null);
102      }
103    }
104  }
105
106  @AfterClass
107  public static void tearDownAfterClass() throws Exception {
108    TEST_UTIL.shutdownMiniCluster();
109  }
110
111  /**
112   * Basic client side validation of HBASE-13262
113   */
114  @Test
115  public void testScannerSeesAllRecords() throws Exception {
116    Connection conn = TEST_UTIL.getConnection();
117    try (Table table = conn.getTable(TABLENAME)) {
118      Scan s = new Scan();
119      s.addFamily(FAMILY);
120      s.setMaxResultSize(-1);
121      s.setBatch(-1);
122      s.setCaching(500);
123      Entry<Long, Long> entry = sumTable(table.getScanner(s));
124      long rowsObserved = entry.getKey();
125      long entriesObserved = entry.getValue();
126
127      // Verify that we see 1M rows and 9M cells
128      assertEquals(NUM_ROWS, rowsObserved);
129      assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
130    }
131  }
132
133  /**
134   * Basic client side validation of HBASE-13262
135   */
136  @Test
137  public void testSmallScannerSeesAllRecords() throws Exception {
138    Connection conn = TEST_UTIL.getConnection();
139    try (Table table = conn.getTable(TABLENAME)) {
140      Scan s = new Scan();
141      s.setReadType(ReadType.PREAD);
142      s.addFamily(FAMILY);
143      s.setMaxResultSize(-1);
144      s.setBatch(-1);
145      s.setCaching(500);
146      Entry<Long, Long> entry = sumTable(table.getScanner(s));
147      long rowsObserved = entry.getKey();
148      long entriesObserved = entry.getValue();
149
150      // Verify that we see 1M rows and 9M cells
151      assertEquals(NUM_ROWS, rowsObserved);
152      assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
153    }
154  }
155
156  /**
157   * Count the number of rows and the number of entries from a scanner The Scanner
158   * @return An entry where the first item is rows observed and the second is entries observed.
159   */
160  private Entry<Long, Long> sumTable(ResultScanner scanner) {
161    long rowsObserved = 0L;
162    long entriesObserved = 0L;
163
164    // Read all the records in the table
165    for (Result result : scanner) {
166      rowsObserved++;
167      while (result.advance()) {
168        entriesObserved++;
169      }
170    }
171    return Maps.immutableEntry(rowsObserved, entriesObserved);
172  }
173}