001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021
022import java.util.ArrayList;
023import java.util.LinkedList;
024import java.util.List;
025import java.util.Map.Entry;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HColumnDescriptor;
030import org.apache.hadoop.hbase.HTableDescriptor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.testclassification.LargeTests;
033import org.apache.hadoop.hbase.util.Bytes;
034import org.junit.AfterClass;
035import org.junit.BeforeClass;
036import org.junit.ClassRule;
037import org.junit.Test;
038import org.junit.experimental.categories.Category;
039import org.slf4j.Logger;
040import org.slf4j.LoggerFactory;
041
042import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
043
044@Category(LargeTests.class)
045public class TestSizeFailures {
046
047  @ClassRule
048  public static final HBaseClassTestRule CLASS_RULE =
049      HBaseClassTestRule.forClass(TestSizeFailures.class);
050
051  private static final Logger LOG = LoggerFactory.getLogger(TestSizeFailures.class);
052  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
053  private static byte [] FAMILY = Bytes.toBytes("testFamily");
054  protected static int SLAVES = 1;
055  private static TableName TABLENAME;
056  private static final int NUM_ROWS = 1000 * 1000, NUM_COLS = 9;
057
058  @BeforeClass
059  public static void setUpBeforeClass() throws Exception {
060    // Uncomment the following lines if more verbosity is needed for
061    // debugging (see HBASE-12285 for details).
062    //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
063    //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
064    //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
065    Configuration conf = TEST_UTIL.getConfiguration();
066    conf.setBoolean("hbase.table.sanity.checks", true); // ignore sanity checks in the server
067    TEST_UTIL.startMiniCluster(SLAVES);
068
069    // Write a bunch of data
070    TABLENAME = TableName.valueOf("testSizeFailures");
071    List<byte[]> qualifiers = new ArrayList<>();
072    for (int i = 1; i <= 10; i++) {
073      qualifiers.add(Bytes.toBytes(Integer.toString(i)));
074    }
075
076    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
077    HTableDescriptor desc = new HTableDescriptor(TABLENAME);
078    desc.addFamily(hcd);
079    byte[][] splits = new byte[9][2];
080    for (int i = 1; i < 10; i++) {
081      int split = 48 + i;
082      splits[i - 1][0] = (byte) (split >>> 8);
083      splits[i - 1][0] = (byte) (split);
084    }
085    TEST_UTIL.getAdmin().createTable(desc, splits);
086    Connection conn = TEST_UTIL.getConnection();
087
088    try (Table table = conn.getTable(TABLENAME)) {
089      List<Put> puts = new LinkedList<>();
090      for (int i = 0; i < NUM_ROWS; i++) {
091        Put p = new Put(Bytes.toBytes(Integer.toString(i)));
092        for (int j = 0; j < NUM_COLS; j++) {
093          byte[] value = new byte[50];
094          Bytes.random(value);
095          p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value);
096        }
097        puts.add(p);
098
099        if (puts.size() == 1000) {
100          table.batch(puts, null);
101          puts.clear();
102        }
103      }
104
105      if (puts.size() > 0) {
106        table.batch(puts, null);
107      }
108    }
109  }
110
111  @AfterClass
112  public static void tearDownAfterClass() throws Exception {
113    TEST_UTIL.shutdownMiniCluster();
114  }
115
116  /**
117   * Basic client side validation of HBASE-13262
118   */
119  @Test
120  public void testScannerSeesAllRecords() throws Exception {
121    Connection conn = TEST_UTIL.getConnection();
122    try (Table table = conn.getTable(TABLENAME)) {
123      Scan s = new Scan();
124      s.addFamily(FAMILY);
125      s.setMaxResultSize(-1);
126      s.setBatch(-1);
127      s.setCaching(500);
128      Entry<Long,Long> entry = sumTable(table.getScanner(s));
129      long rowsObserved = entry.getKey();
130      long entriesObserved = entry.getValue();
131
132      // Verify that we see 1M rows and 9M cells
133      assertEquals(NUM_ROWS, rowsObserved);
134      assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
135    }
136  }
137
138  /**
139   * Basic client side validation of HBASE-13262
140   */
141  @Test
142  public void testSmallScannerSeesAllRecords() throws Exception {
143    Connection conn = TEST_UTIL.getConnection();
144    try (Table table = conn.getTable(TABLENAME)) {
145      Scan s = new Scan();
146      s.setSmall(true);
147      s.addFamily(FAMILY);
148      s.setMaxResultSize(-1);
149      s.setBatch(-1);
150      s.setCaching(500);
151      Entry<Long,Long> entry = sumTable(table.getScanner(s));
152      long rowsObserved = entry.getKey();
153      long entriesObserved = entry.getValue();
154
155      // Verify that we see 1M rows and 9M cells
156      assertEquals(NUM_ROWS, rowsObserved);
157      assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
158    }
159  }
160
161  /**
162   * Count the number of rows and the number of entries from a scanner
163   *
164   * @param scanner
165   *          The Scanner
166   * @return An entry where the first item is rows observed and the second is entries observed.
167   */
168  private Entry<Long,Long> sumTable(ResultScanner scanner) {
169    long rowsObserved = 0L;
170    long entriesObserved = 0L;
171
172    // Read all the records in the table
173    for (Result result : scanner) {
174      rowsObserved++;
175      while (result.advance()) {
176        entriesObserved++;
177      }
178    }
179    return Maps.immutableEntry(rowsObserved,entriesObserved);
180  }
181}