001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.filter; 019 020import static org.junit.Assert.assertEquals; 021 022import java.io.IOException; 023import java.util.ArrayList; 024import java.util.List; 025import org.apache.hadoop.hbase.Cell; 026import org.apache.hadoop.hbase.HBaseClassTestRule; 027import org.apache.hadoop.hbase.HBaseTestingUtility; 028import org.apache.hadoop.hbase.TableName; 029import org.apache.hadoop.hbase.client.Durability; 030import org.apache.hadoop.hbase.client.Put; 031import org.apache.hadoop.hbase.client.Result; 032import org.apache.hadoop.hbase.client.ResultScanner; 033import org.apache.hadoop.hbase.client.Scan; 034import org.apache.hadoop.hbase.client.Table; 035import org.apache.hadoop.hbase.filter.MultiRowRangeFilter.RowRange; 036import org.apache.hadoop.hbase.io.hfile.HFile; 037import org.apache.hadoop.hbase.testclassification.MediumTests; 038import org.apache.hadoop.hbase.util.Bytes; 039import org.junit.AfterClass; 040import org.junit.BeforeClass; 041import org.junit.ClassRule; 042import org.junit.Rule; 043import org.junit.Test; 044import org.junit.experimental.categories.Category; 045import org.junit.rules.TestName; 046import org.slf4j.Logger; 047import org.slf4j.LoggerFactory; 048 049/* 050 * This test is for the optimization added in HBASE-15243. 051 * FilterList with two MultiRowRangeFilter's is constructed using Operator.MUST_PASS_ONE. 052 */ 053@Category(MediumTests.class) 054public class TestFilterListOrOperatorWithBlkCnt { 055 056 @ClassRule 057 public static final HBaseClassTestRule CLASS_RULE = 058 HBaseClassTestRule.forClass(TestFilterListOrOperatorWithBlkCnt.class); 059 060 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 061 private static final Logger LOG = 062 LoggerFactory.getLogger(TestFilterListOrOperatorWithBlkCnt.class); 063 private byte[] family = Bytes.toBytes("family"); 064 private byte[] qf = Bytes.toBytes("qf"); 065 private byte[] value = Bytes.toBytes("val"); 066 private TableName tableName; 067 private int numRows = 10000; 068 069 @Rule 070 public TestName name = new TestName(); 071 072 /** 073 * n 074 */ 075 @BeforeClass 076 public static void setUpBeforeClass() throws Exception { 077 long blkSize = 4096; 078 /* 079 * dfs block size is adjusted so that the specified number of rows would result in multiple 080 * blocks (8 for this test). Later in the test, assertion is made on the number of blocks read. 081 */ 082 TEST_UTIL.getConfiguration().setLong("dfs.blocksize", blkSize); 083 TEST_UTIL.getConfiguration().setLong("dfs.bytes-per-checksum", blkSize); 084 TEST_UTIL.startMiniCluster(); 085 } 086 087 /** 088 * n 089 */ 090 @AfterClass 091 public static void tearDownAfterClass() throws Exception { 092 TEST_UTIL.shutdownMiniCluster(); 093 } 094 095 private static long getBlkAccessCount() { 096 return HFile.DATABLOCK_READ_COUNT.sum(); 097 } 098 099 @Test 100 public void testMultiRowRangeWithFilterListOrOperatorWithBlkCnt() throws IOException { 101 tableName = TableName.valueOf(name.getMethodName()); 102 Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); 103 generateRows(numRows, ht, family, qf, value); 104 105 Scan scan = new Scan(); 106 scan.setMaxVersions(); 107 long blocksStart = getBlkAccessCount(); 108 109 List<RowRange> ranges1 = new ArrayList<>(); 110 ranges1.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(15), false)); 111 ranges1.add(new RowRange(Bytes.toBytes(9980), true, Bytes.toBytes(9985), false)); 112 113 MultiRowRangeFilter filter1 = new MultiRowRangeFilter(ranges1); 114 115 List<RowRange> ranges2 = new ArrayList<>(); 116 ranges2.add(new RowRange(Bytes.toBytes(15), true, Bytes.toBytes(20), false)); 117 ranges2.add(new RowRange(Bytes.toBytes(9985), true, Bytes.toBytes(9990), false)); 118 119 MultiRowRangeFilter filter2 = new MultiRowRangeFilter(ranges2); 120 121 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); 122 filterList.addFilter(filter1); 123 filterList.addFilter(filter2); 124 scan.setFilter(filterList); 125 int resultsSize = getResultsSize(ht, scan); 126 LOG.info("found " + resultsSize + " results"); 127 List<Cell> results1 = getScanResult(Bytes.toBytes(10), Bytes.toBytes(20), ht); 128 List<Cell> results2 = getScanResult(Bytes.toBytes(9980), Bytes.toBytes(9990), ht); 129 130 assertEquals(results1.size() + results2.size(), resultsSize); 131 long blocksEnd = getBlkAccessCount(); 132 long diff = blocksEnd - blocksStart; 133 LOG.info("Diff in number of blocks " + diff); 134 /* 135 * Verify that we don't read all the blocks (8 in total). 136 */ 137 assertEquals(4, diff); 138 139 ht.close(); 140 } 141 142 private void generateRows(int numberOfRows, Table ht, byte[] family, byte[] qf, byte[] value) 143 throws IOException { 144 for (int i = 0; i < numberOfRows; i++) { 145 byte[] row = Bytes.toBytes(i); 146 Put p = new Put(row); 147 p.addColumn(family, qf, value); 148 p.setDurability(Durability.SKIP_WAL); 149 ht.put(p); 150 } 151 TEST_UTIL.flush(); 152 } 153 154 private List<Cell> getScanResult(byte[] startRow, byte[] stopRow, Table ht) throws IOException { 155 Scan scan = new Scan(); 156 scan.setMaxVersions(); 157 if (!Bytes.toString(startRow).isEmpty()) { 158 scan.setStartRow(startRow); 159 } 160 if (!Bytes.toString(stopRow).isEmpty()) { 161 scan.setStopRow(stopRow); 162 } 163 ResultScanner scanner = ht.getScanner(scan); 164 List<Cell> kvList = new ArrayList<>(); 165 Result r; 166 while ((r = scanner.next()) != null) { 167 for (Cell kv : r.listCells()) { 168 kvList.add(kv); 169 } 170 } 171 return kvList; 172 } 173 174 private int getResultsSize(Table ht, Scan scan) throws IOException { 175 ResultScanner scanner = ht.getScanner(scan); 176 List<Cell> results = new ArrayList<>(); 177 Result r; 178 while ((r = scanner.next()) != null) { 179 for (Cell kv : r.listCells()) { 180 results.add(kv); 181 } 182 } 183 return results.size(); 184 } 185}