001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021
022import java.util.ArrayList;
023import java.util.LinkedList;
024import java.util.List;
025import java.util.Map.Entry;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HColumnDescriptor;
030import org.apache.hadoop.hbase.HTableDescriptor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.testclassification.LargeTests;
033import org.apache.hadoop.hbase.util.Bytes;
034import org.junit.AfterClass;
035import org.junit.BeforeClass;
036import org.junit.ClassRule;
037import org.junit.Test;
038import org.junit.experimental.categories.Category;
039import org.slf4j.Logger;
040import org.slf4j.LoggerFactory;
041
042import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
043
044@Category(LargeTests.class)
045public class TestSizeFailures {
046
047  @ClassRule
048  public static final HBaseClassTestRule CLASS_RULE =
049    HBaseClassTestRule.forClass(TestSizeFailures.class);
050
051  private static final Logger LOG = LoggerFactory.getLogger(TestSizeFailures.class);
052  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
053  private static byte[] FAMILY = Bytes.toBytes("testFamily");
054  protected static int SLAVES = 1;
055  private static TableName TABLENAME;
056  private static final int NUM_ROWS = 1000 * 1000, NUM_COLS = 9;
057
058  @BeforeClass
059  public static void setUpBeforeClass() throws Exception {
060    // Uncomment the following lines if more verbosity is needed for
061    // debugging (see HBASE-12285 for details).
062    // ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
063    // ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
064    // ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
065    Configuration conf = TEST_UTIL.getConfiguration();
066    TEST_UTIL.startMiniCluster(SLAVES);
067
068    // Write a bunch of data
069    TABLENAME = TableName.valueOf("testSizeFailures");
070    List<byte[]> qualifiers = new ArrayList<>();
071    for (int i = 1; i <= 10; i++) {
072      qualifiers.add(Bytes.toBytes(Integer.toString(i)));
073    }
074
075    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
076    HTableDescriptor desc = new HTableDescriptor(TABLENAME);
077    desc.addFamily(hcd);
078    byte[][] splits = new byte[9][2];
079    for (int i = 1; i < 10; i++) {
080      int split = 48 + i;
081      splits[i - 1][0] = (byte) (split >>> 8);
082      splits[i - 1][0] = (byte) (split);
083    }
084    TEST_UTIL.getAdmin().createTable(desc, splits);
085    Connection conn = TEST_UTIL.getConnection();
086
087    try (Table table = conn.getTable(TABLENAME)) {
088      List<Put> puts = new LinkedList<>();
089      for (int i = 0; i < NUM_ROWS; i++) {
090        Put p = new Put(Bytes.toBytes(Integer.toString(i)));
091        for (int j = 0; j < NUM_COLS; j++) {
092          byte[] value = new byte[50];
093          Bytes.random(value);
094          p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value);
095        }
096        puts.add(p);
097
098        if (puts.size() == 1000) {
099          table.batch(puts, null);
100          puts.clear();
101        }
102      }
103
104      if (puts.size() > 0) {
105        table.batch(puts, null);
106      }
107    }
108  }
109
110  @AfterClass
111  public static void tearDownAfterClass() throws Exception {
112    TEST_UTIL.shutdownMiniCluster();
113  }
114
115  /**
116   * Basic client side validation of HBASE-13262
117   */
118  @Test
119  public void testScannerSeesAllRecords() throws Exception {
120    Connection conn = TEST_UTIL.getConnection();
121    try (Table table = conn.getTable(TABLENAME)) {
122      Scan s = new Scan();
123      s.addFamily(FAMILY);
124      s.setMaxResultSize(-1);
125      s.setBatch(-1);
126      s.setCaching(500);
127      Entry<Long, Long> entry = sumTable(table.getScanner(s));
128      long rowsObserved = entry.getKey();
129      long entriesObserved = entry.getValue();
130
131      // Verify that we see 1M rows and 9M cells
132      assertEquals(NUM_ROWS, rowsObserved);
133      assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
134    }
135  }
136
137  /**
138   * Basic client side validation of HBASE-13262
139   */
140  @Test
141  public void testSmallScannerSeesAllRecords() throws Exception {
142    Connection conn = TEST_UTIL.getConnection();
143    try (Table table = conn.getTable(TABLENAME)) {
144      Scan s = new Scan();
145      s.setSmall(true);
146      s.addFamily(FAMILY);
147      s.setMaxResultSize(-1);
148      s.setBatch(-1);
149      s.setCaching(500);
150      Entry<Long, Long> entry = sumTable(table.getScanner(s));
151      long rowsObserved = entry.getKey();
152      long entriesObserved = entry.getValue();
153
154      // Verify that we see 1M rows and 9M cells
155      assertEquals(NUM_ROWS, rowsObserved);
156      assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
157    }
158  }
159
160  /**
161   * Count the number of rows and the number of entries from a scanner n * The Scanner
162   * @return An entry where the first item is rows observed and the second is entries observed.
163   */
164  private Entry<Long, Long> sumTable(ResultScanner scanner) {
165    long rowsObserved = 0L;
166    long entriesObserved = 0L;
167
168    // Read all the records in the table
169    for (Result result : scanner) {
170      rowsObserved++;
171      while (result.advance()) {
172        entriesObserved++;
173      }
174    }
175    return Maps.immutableEntry(rowsObserved, entriesObserved);
176  }
177}