001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021
022import java.util.ArrayList;
023import java.util.LinkedList;
024import java.util.List;
025import java.util.Map.Entry;
026import org.apache.hadoop.hbase.HBaseClassTestRule;
027import org.apache.hadoop.hbase.HBaseTestingUtility;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.testclassification.LargeTests;
030import org.apache.hadoop.hbase.util.Bytes;
031import org.junit.AfterClass;
032import org.junit.BeforeClass;
033import org.junit.ClassRule;
034import org.junit.Test;
035import org.junit.experimental.categories.Category;
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038
039import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
040
041@Category(LargeTests.class)
042public class TestSizeFailures {
043
044  @ClassRule
045  public static final HBaseClassTestRule CLASS_RULE =
046      HBaseClassTestRule.forClass(TestSizeFailures.class);
047
048  private static final Logger LOG = LoggerFactory.getLogger(TestSizeFailures.class);
049  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
050  private static byte [] FAMILY = Bytes.toBytes("testFamily");
051  protected static int SLAVES = 1;
052  private static TableName TABLENAME;
053  private static final int NUM_ROWS = 1000 * 1000, NUM_COLS = 9;
054
055  @BeforeClass
056  public static void setUpBeforeClass() throws Exception {
057    // Uncomment the following lines if more verbosity is needed for
058    // debugging (see HBASE-12285 for details).
059    //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
060    //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
061    //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
062    TEST_UTIL.startMiniCluster(SLAVES);
063
064    // Write a bunch of data
065    TABLENAME = TableName.valueOf("testSizeFailures");
066    List<byte[]> qualifiers = new ArrayList<>();
067    for (int i = 1; i <= 10; i++) {
068      qualifiers.add(Bytes.toBytes(Integer.toString(i)));
069    }
070
071    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TABLENAME)
072      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build();
073    byte[][] splits = new byte[9][2];
074    for (int i = 1; i < 10; i++) {
075      int split = 48 + i;
076      splits[i - 1][0] = (byte) (split >>> 8);
077      splits[i - 1][0] = (byte) (split);
078    }
079    TEST_UTIL.getAdmin().createTable(tableDescriptor, splits);
080    Connection conn = TEST_UTIL.getConnection();
081
082    try (Table table = conn.getTable(TABLENAME)) {
083      List<Put> puts = new LinkedList<>();
084      for (int i = 0; i < NUM_ROWS; i++) {
085        Put p = new Put(Bytes.toBytes(Integer.toString(i)));
086        for (int j = 0; j < NUM_COLS; j++) {
087          byte[] value = new byte[50];
088          Bytes.random(value);
089          p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value);
090        }
091        puts.add(p);
092
093        if (puts.size() == 1000) {
094          table.batch(puts, null);
095          puts.clear();
096        }
097      }
098
099      if (puts.size() > 0) {
100        table.batch(puts, null);
101      }
102    }
103  }
104
105  @AfterClass
106  public static void tearDownAfterClass() throws Exception {
107    TEST_UTIL.shutdownMiniCluster();
108  }
109
110  /**
111   * Basic client side validation of HBASE-13262
112   */
113  @Test
114  public void testScannerSeesAllRecords() throws Exception {
115    Connection conn = TEST_UTIL.getConnection();
116    try (Table table = conn.getTable(TABLENAME)) {
117      Scan s = new Scan();
118      s.addFamily(FAMILY);
119      s.setMaxResultSize(-1);
120      s.setBatch(-1);
121      s.setCaching(500);
122      Entry<Long,Long> entry = sumTable(table.getScanner(s));
123      long rowsObserved = entry.getKey();
124      long entriesObserved = entry.getValue();
125
126      // Verify that we see 1M rows and 9M cells
127      assertEquals(NUM_ROWS, rowsObserved);
128      assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
129    }
130  }
131
132  /**
133   * Basic client side validation of HBASE-13262
134   */
135  @Test
136  public void testSmallScannerSeesAllRecords() throws Exception {
137    Connection conn = TEST_UTIL.getConnection();
138    try (Table table = conn.getTable(TABLENAME)) {
139      Scan s = new Scan();
140      s.setSmall(true);
141      s.addFamily(FAMILY);
142      s.setMaxResultSize(-1);
143      s.setBatch(-1);
144      s.setCaching(500);
145      Entry<Long,Long> entry = sumTable(table.getScanner(s));
146      long rowsObserved = entry.getKey();
147      long entriesObserved = entry.getValue();
148
149      // Verify that we see 1M rows and 9M cells
150      assertEquals(NUM_ROWS, rowsObserved);
151      assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
152    }
153  }
154
155  /**
156   * Count the number of rows and the number of entries from a scanner
157   *
158   * @param scanner
159   *          The Scanner
160   * @return An entry where the first item is rows observed and the second is entries observed.
161   */
162  private Entry<Long,Long> sumTable(ResultScanner scanner) {
163    long rowsObserved = 0L;
164    long entriesObserved = 0L;
165
166    // Read all the records in the table
167    for (Result result : scanner) {
168      rowsObserved++;
169      while (result.advance()) {
170        entriesObserved++;
171      }
172    }
173    return Maps.immutableEntry(rowsObserved,entriesObserved);
174  }
175}