001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.List; 025import org.apache.hadoop.hbase.Cell; 026import org.apache.hadoop.hbase.HBaseClassTestRule; 027import org.apache.hadoop.hbase.HBaseTestingUtil; 028import org.apache.hadoop.hbase.TableName; 029import org.apache.hadoop.hbase.client.Durability; 030import org.apache.hadoop.hbase.client.Put; 031import org.apache.hadoop.hbase.client.Result; 032import org.apache.hadoop.hbase.client.ResultScanner; 033import org.apache.hadoop.hbase.client.Scan; 034import org.apache.hadoop.hbase.client.Table; 035import org.apache.hadoop.hbase.filter.MultiRowRangeFilter.RowRange; 036import org.apache.hadoop.hbase.io.hfile.HFile; 037import org.apache.hadoop.hbase.testclassification.MediumTests; 038import org.apache.hadoop.hbase.util.Bytes; 039import org.junit.AfterClass; 040import org.junit.BeforeClass; 041import org.junit.ClassRule; 042import org.junit.Rule; 043import org.junit.Test; 044import org.junit.experimental.categories.Category; 045import org.junit.rules.TestName; 046import org.slf4j.Logger; 047import org.slf4j.LoggerFactory; 048 049/* 050 * This test is for the optimization added in HBASE-15243. 051 * FilterList with two MultiRowRangeFilter's is constructed using Operator.MUST_PASS_ONE. 052 */ 053@Category(MediumTests.class) 054public class TestFilterListOrOperatorWithBlkCnt { 055 056 @ClassRule 057 public static final HBaseClassTestRule CLASS_RULE = 058 HBaseClassTestRule.forClass(TestFilterListOrOperatorWithBlkCnt.class); 059 060 private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 061 private static final Logger LOG = 062 LoggerFactory.getLogger(TestFilterListOrOperatorWithBlkCnt.class); 063 private byte[] family = Bytes.toBytes("family"); 064 private byte[] qf = Bytes.toBytes("qf"); 065 private byte[] value = Bytes.toBytes("val"); 066 private TableName tableName; 067 private int numRows = 10000; 068 069 @Rule 070 public TestName name = new TestName(); 071 072 /** 073 * */ 074 @BeforeClass 075 public static void setUpBeforeClass() throws Exception { 076 long blkSize = 4096; 077 /* 078 * dfs block size is adjusted so that the specified number of rows would result in multiple 079 * blocks (8 for this test). Later in the test, assertion is made on the number of blocks read. 080 */ 081 TEST_UTIL.getConfiguration().setLong("dfs.blocksize", blkSize); 082 TEST_UTIL.getConfiguration().setLong("dfs.bytes-per-checksum", blkSize); 083 TEST_UTIL.startMiniCluster(); 084 } 085 086 /** 087 * */ 088 @AfterClass 089 public static void tearDownAfterClass() throws Exception { 090 TEST_UTIL.shutdownMiniCluster(); 091 } 092 093 private static long getBlkAccessCount() { 094 return HFile.DATABLOCK_READ_COUNT.sum(); 095 } 096 097 @Test 098 public void testMultiRowRangeWithFilterListOrOperatorWithBlkCnt() throws IOException { 099 tableName = TableName.valueOf(name.getMethodName()); 100 Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); 101 generateRows(numRows, ht, family, qf, value); 102 103 Scan scan = new Scan(); 104 scan.readAllVersions(); 105 long blocksStart = getBlkAccessCount(); 106 107 List<RowRange> ranges1 = new ArrayList<>(); 108 ranges1.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(15), false)); 109 ranges1.add(new RowRange(Bytes.toBytes(9980), true, Bytes.toBytes(9985), false)); 110 111 MultiRowRangeFilter filter1 = new MultiRowRangeFilter(ranges1); 112 113 List<RowRange> ranges2 = new ArrayList<>(); 114 ranges2.add(new RowRange(Bytes.toBytes(15), true, Bytes.toBytes(20), false)); 115 ranges2.add(new RowRange(Bytes.toBytes(9985), true, Bytes.toBytes(9990), false)); 116 117 MultiRowRangeFilter filter2 = new MultiRowRangeFilter(ranges2); 118 119 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); 120 filterList.addFilter(filter1); 121 filterList.addFilter(filter2); 122 scan.setFilter(filterList); 123 int resultsSize = getResultsSize(ht, scan); 124 LOG.info("found " + resultsSize + " results"); 125 List<Cell> results1 = getScanResult(Bytes.toBytes(10), Bytes.toBytes(20), ht); 126 List<Cell> results2 = getScanResult(Bytes.toBytes(9980), Bytes.toBytes(9990), ht); 127 128 assertEquals(results1.size() + results2.size(), resultsSize); 129 long blocksEnd = getBlkAccessCount(); 130 long diff = blocksEnd - blocksStart; 131 LOG.info("Diff in number of blocks " + diff); 132 /* 133 * Verify that we don't read all the blocks (8 in total). 134 */ 135 assertEquals(4, diff); 136 137 ht.close(); 138 } 139 140 private void generateRows(int numberOfRows, Table ht, byte[] family, byte[] qf, byte[] value) 141 throws IOException { 142 for (int i = 0; i < numberOfRows; i++) { 143 byte[] row = Bytes.toBytes(i); 144 Put p = new Put(row); 145 p.addColumn(family, qf, value); 146 p.setDurability(Durability.SKIP_WAL); 147 ht.put(p); 148 } 149 TEST_UTIL.flush(); 150 } 151 152 private List<Cell> getScanResult(byte[] startRow, byte[] stopRow, Table ht) throws IOException { 153 Scan scan = new Scan(); 154 scan.readAllVersions(); 155 if (!Bytes.toString(startRow).isEmpty()) { 156 scan.withStartRow(startRow); 157 } 158 if (!Bytes.toString(stopRow).isEmpty()) { 159 scan.withStopRow(stopRow); 160 } 161 ResultScanner scanner = ht.getScanner(scan); 162 List<Cell> kvList = new ArrayList<>(); 163 Result r; 164 while ((r = scanner.next()) != null) { 165 for (Cell kv : r.listCells()) { 166 kvList.add(kv); 167 } 168 } 169 return kvList; 170 } 171 172 private int getResultsSize(Table ht, Scan scan) throws IOException { 173 ResultScanner scanner = ht.getScanner(scan); 174 List<Cell> results = new ArrayList<>(); 175 Result r; 176 while ((r = scanner.next()) != null) { 177 for (Cell kv : r.listCells()) { 178 results.add(kv); 179 } 180 } 181 return results.size(); 182 } 183}