001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.client; 019 020import static org.junit.Assert.assertEquals; 021 022import java.util.ArrayList; 023import java.util.LinkedList; 024import java.util.List; 025import java.util.Map.Entry; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.hbase.HBaseClassTestRule; 028import org.apache.hadoop.hbase.HBaseTestingUtility; 029import org.apache.hadoop.hbase.HColumnDescriptor; 030import org.apache.hadoop.hbase.HTableDescriptor; 031import org.apache.hadoop.hbase.TableName; 032import org.apache.hadoop.hbase.testclassification.LargeTests; 033import org.apache.hadoop.hbase.util.Bytes; 034import org.junit.AfterClass; 035import org.junit.BeforeClass; 036import org.junit.ClassRule; 037import org.junit.Test; 038import org.junit.experimental.categories.Category; 039import org.slf4j.Logger; 040import org.slf4j.LoggerFactory; 041 042import org.apache.hbase.thirdparty.com.google.common.collect.Maps; 043 044@Category(LargeTests.class) 045public class TestSizeFailures { 046 047 @ClassRule 048 public static final HBaseClassTestRule CLASS_RULE = 049 HBaseClassTestRule.forClass(TestSizeFailures.class); 050 051 private static final Logger LOG = LoggerFactory.getLogger(TestSizeFailures.class); 052 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 053 private static byte [] FAMILY = Bytes.toBytes("testFamily"); 054 protected static int SLAVES = 1; 055 private static TableName TABLENAME; 056 private static final int NUM_ROWS = 1000 * 1000, NUM_COLS = 9; 057 058 @BeforeClass 059 public static void setUpBeforeClass() throws Exception { 060 // Uncomment the following lines if more verbosity is needed for 061 // debugging (see HBASE-12285 for details). 062 //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); 063 //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); 064 //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); 065 Configuration conf = TEST_UTIL.getConfiguration(); 066 conf.setBoolean("hbase.table.sanity.checks", true); // ignore sanity checks in the server 067 TEST_UTIL.startMiniCluster(SLAVES); 068 069 // Write a bunch of data 070 TABLENAME = TableName.valueOf("testSizeFailures"); 071 List<byte[]> qualifiers = new ArrayList<>(); 072 for (int i = 1; i <= 10; i++) { 073 qualifiers.add(Bytes.toBytes(Integer.toString(i))); 074 } 075 076 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); 077 HTableDescriptor desc = new HTableDescriptor(TABLENAME); 078 desc.addFamily(hcd); 079 byte[][] splits = new byte[9][2]; 080 for (int i = 1; i < 10; i++) { 081 int split = 48 + i; 082 splits[i - 1][0] = (byte) (split >>> 8); 083 splits[i - 1][0] = (byte) (split); 084 } 085 TEST_UTIL.getAdmin().createTable(desc, splits); 086 Connection conn = TEST_UTIL.getConnection(); 087 088 try (Table table = conn.getTable(TABLENAME)) { 089 List<Put> puts = new LinkedList<>(); 090 for (int i = 0; i < NUM_ROWS; i++) { 091 Put p = new Put(Bytes.toBytes(Integer.toString(i))); 092 for (int j = 0; j < NUM_COLS; j++) { 093 byte[] value = new byte[50]; 094 Bytes.random(value); 095 p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value); 096 } 097 puts.add(p); 098 099 if (puts.size() == 1000) { 100 table.batch(puts, null); 101 puts.clear(); 102 } 103 } 104 105 if (puts.size() > 0) { 106 table.batch(puts, null); 107 } 108 } 109 } 110 111 @AfterClass 112 public static void tearDownAfterClass() throws Exception { 113 TEST_UTIL.shutdownMiniCluster(); 114 } 115 116 /** 117 * Basic client side validation of HBASE-13262 118 */ 119 @Test 120 public void testScannerSeesAllRecords() throws Exception { 121 Connection conn = TEST_UTIL.getConnection(); 122 try (Table table = conn.getTable(TABLENAME)) { 123 Scan s = new Scan(); 124 s.addFamily(FAMILY); 125 s.setMaxResultSize(-1); 126 s.setBatch(-1); 127 s.setCaching(500); 128 Entry<Long,Long> entry = sumTable(table.getScanner(s)); 129 long rowsObserved = entry.getKey(); 130 long entriesObserved = entry.getValue(); 131 132 // Verify that we see 1M rows and 9M cells 133 assertEquals(NUM_ROWS, rowsObserved); 134 assertEquals(NUM_ROWS * NUM_COLS, entriesObserved); 135 } 136 } 137 138 /** 139 * Basic client side validation of HBASE-13262 140 */ 141 @Test 142 public void testSmallScannerSeesAllRecords() throws Exception { 143 Connection conn = TEST_UTIL.getConnection(); 144 try (Table table = conn.getTable(TABLENAME)) { 145 Scan s = new Scan(); 146 s.setSmall(true); 147 s.addFamily(FAMILY); 148 s.setMaxResultSize(-1); 149 s.setBatch(-1); 150 s.setCaching(500); 151 Entry<Long,Long> entry = sumTable(table.getScanner(s)); 152 long rowsObserved = entry.getKey(); 153 long entriesObserved = entry.getValue(); 154 155 // Verify that we see 1M rows and 9M cells 156 assertEquals(NUM_ROWS, rowsObserved); 157 assertEquals(NUM_ROWS * NUM_COLS, entriesObserved); 158 } 159 } 160 161 /** 162 * Count the number of rows and the number of entries from a scanner 163 * 164 * @param scanner 165 * The Scanner 166 * @return An entry where the first item is rows observed and the second is entries observed. 167 */ 168 private Entry<Long,Long> sumTable(ResultScanner scanner) { 169 long rowsObserved = 0L; 170 long entriesObserved = 0L; 171 172 // Read all the records in the table 173 for (Result result : scanner) { 174 rowsObserved++; 175 while (result.advance()) { 176 entriesObserved++; 177 } 178 } 179 return Maps.immutableEntry(rowsObserved,entriesObserved); 180 } 181}