001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.List; 025import org.apache.hadoop.hbase.Cell; 026import org.apache.hadoop.hbase.HBaseClassTestRule; 027import org.apache.hadoop.hbase.HBaseTestingUtility; 028import org.apache.hadoop.hbase.TableName; 029import org.apache.hadoop.hbase.client.Durability; 030import org.apache.hadoop.hbase.client.Put; 031import org.apache.hadoop.hbase.client.Result; 032import org.apache.hadoop.hbase.client.ResultScanner; 033import org.apache.hadoop.hbase.client.Scan; 034import org.apache.hadoop.hbase.client.Table; 035import org.apache.hadoop.hbase.filter.MultiRowRangeFilter.RowRange; 036import org.apache.hadoop.hbase.io.hfile.HFile; 037import org.apache.hadoop.hbase.testclassification.MediumTests; 038import org.apache.hadoop.hbase.util.Bytes; 039import org.junit.AfterClass; 040import org.junit.BeforeClass; 041import org.junit.ClassRule; 042import org.junit.Rule; 043import org.junit.Test; 044import org.junit.experimental.categories.Category; 045import org.junit.rules.TestName; 046import org.slf4j.Logger; 047import org.slf4j.LoggerFactory; 048 049/* 050 * This test is for the optimization added in HBASE-15243. 051 * FilterList with two MultiRowRangeFilter's is constructed using Operator.MUST_PASS_ONE. 052 */ 053@Category(MediumTests.class) 054public class TestFilterListOrOperatorWithBlkCnt { 055 056 @ClassRule 057 public static final HBaseClassTestRule CLASS_RULE = 058 HBaseClassTestRule.forClass(TestFilterListOrOperatorWithBlkCnt.class); 059 060 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 061 private static final Logger LOG = 062 LoggerFactory.getLogger(TestFilterListOrOperatorWithBlkCnt.class); 063 private byte[] family = Bytes.toBytes("family"); 064 private byte[] qf = Bytes.toBytes("qf"); 065 private byte[] value = Bytes.toBytes("val"); 066 private TableName tableName; 067 private int numRows = 10000; 068 069 @Rule 070 public TestName name = new TestName(); 071 072 /** 073 * @throws Exception 074 */ 075 @BeforeClass 076 public static void setUpBeforeClass() throws Exception { 077 long blkSize = 4096; 078 /* 079 * dfs block size is adjusted so that the specified number of rows would result in 080 * multiple blocks (8 for this test). 081 * Later in the test, assertion is made on the number of blocks read. 082 */ 083 TEST_UTIL.getConfiguration().setLong("dfs.blocksize", blkSize); 084 TEST_UTIL.getConfiguration().setLong("dfs.bytes-per-checksum", blkSize); 085 TEST_UTIL.startMiniCluster(); 086 } 087 088 /** 089 * @throws Exception 090 */ 091 @AfterClass 092 public static void tearDownAfterClass() throws Exception { 093 TEST_UTIL.shutdownMiniCluster(); 094 } 095 096 private static long getBlkAccessCount() { 097 return HFile.DATABLOCK_READ_COUNT.sum(); 098 } 099 100 @Test 101 public void testMultiRowRangeWithFilterListOrOperatorWithBlkCnt() throws IOException { 102 tableName = TableName.valueOf(name.getMethodName()); 103 Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); 104 generateRows(numRows, ht, family, qf, value); 105 106 Scan scan = new Scan(); 107 scan.setMaxVersions(); 108 long blocksStart = getBlkAccessCount(); 109 110 List<RowRange> ranges1 = new ArrayList<>(); 111 ranges1.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(15), false)); 112 ranges1.add(new RowRange(Bytes.toBytes(9980), true, Bytes.toBytes(9985), false)); 113 114 MultiRowRangeFilter filter1 = new MultiRowRangeFilter(ranges1); 115 116 List<RowRange> ranges2 = new ArrayList<>(); 117 ranges2.add(new RowRange(Bytes.toBytes(15), true, Bytes.toBytes(20), false)); 118 ranges2.add(new RowRange(Bytes.toBytes(9985), true, Bytes.toBytes(9990), false)); 119 120 MultiRowRangeFilter filter2 = new MultiRowRangeFilter(ranges2); 121 122 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); 123 filterList.addFilter(filter1); 124 filterList.addFilter(filter2); 125 scan.setFilter(filterList); 126 int resultsSize = getResultsSize(ht, scan); 127 LOG.info("found " + resultsSize + " results"); 128 List<Cell> results1 = getScanResult(Bytes.toBytes(10), Bytes.toBytes(20), ht); 129 List<Cell> results2 = getScanResult(Bytes.toBytes(9980), Bytes.toBytes(9990), ht); 130 131 assertEquals(results1.size() + results2.size(), resultsSize); 132 long blocksEnd = getBlkAccessCount(); 133 long diff = blocksEnd - blocksStart; 134 LOG.info("Diff in number of blocks " + diff); 135 /* 136 * Verify that we don't read all the blocks (8 in total). 137 */ 138 assertEquals(4, diff); 139 140 ht.close(); 141 } 142 143 private void generateRows(int numberOfRows, Table ht, byte[] family, byte[] qf, byte[] value) 144 throws IOException { 145 for (int i = 0; i < numberOfRows; i++) { 146 byte[] row = Bytes.toBytes(i); 147 Put p = new Put(row); 148 p.addColumn(family, qf, value); 149 p.setDurability(Durability.SKIP_WAL); 150 ht.put(p); 151 } 152 TEST_UTIL.flush(); 153 } 154 155 private List<Cell> getScanResult(byte[] startRow, byte[] stopRow, Table ht) throws IOException { 156 Scan scan = new Scan(); 157 scan.setMaxVersions(); 158 if(!Bytes.toString(startRow).isEmpty()) { 159 scan.setStartRow(startRow); 160 } 161 if(!Bytes.toString(stopRow).isEmpty()) { 162 scan.setStopRow(stopRow); 163 } 164 ResultScanner scanner = ht.getScanner(scan); 165 List<Cell> kvList = new ArrayList<>(); 166 Result r; 167 while ((r = scanner.next()) != null) { 168 for (Cell kv : r.listCells()) { 169 kvList.add(kv); 170 } 171 } 172 return kvList; 173 } 174 175 private int getResultsSize(Table ht, Scan scan) throws IOException { 176 ResultScanner scanner = ht.getScanner(scan); 177 List<Cell> results = new ArrayList<>(); 178 Result r; 179 while ((r = scanner.next()) != null) { 180 for (Cell kv : r.listCells()) { 181 results.add(kv); 182 } 183 } 184 return results.size(); 185 } 186}