001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Collection;
027import java.util.Collections;
028import java.util.Comparator;
029import java.util.List;
030import java.util.NavigableSet;
031import java.util.TreeSet;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.Cell;
036import org.apache.hadoop.hbase.CellUtil;
037import org.apache.hadoop.hbase.HBaseClassTestRule;
038import org.apache.hadoop.hbase.HBaseTestingUtility;
039import org.apache.hadoop.hbase.HColumnDescriptor;
040import org.apache.hadoop.hbase.KeyValue;
041import org.apache.hadoop.hbase.KeyValueTestUtil;
042import org.apache.hadoop.hbase.client.Put;
043import org.apache.hadoop.hbase.client.Scan;
044import org.apache.hadoop.hbase.io.compress.Compression;
045import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
046import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
047import org.apache.hadoop.hbase.testclassification.RegionServerTests;
048import org.apache.hadoop.hbase.testclassification.SmallTests;
049import org.apache.hadoop.hbase.util.BloomFilterUtil;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.junit.Before;
052import org.junit.ClassRule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.runner.RunWith;
056import org.junit.runners.Parameterized;
057import org.junit.runners.Parameterized.Parameters;
058import org.slf4j.Logger;
059import org.slf4j.LoggerFactory;
060
061/**
062 * Test a multi-column scanner when there is a Bloom filter false-positive.
063 * This is needed for the multi-column Bloom filter optimization.
064 */
065@RunWith(Parameterized.class)
066@Category({RegionServerTests.class, SmallTests.class})
067public class TestScanWithBloomError {
068
069  @ClassRule
070  public static final HBaseClassTestRule CLASS_RULE =
071      HBaseClassTestRule.forClass(TestScanWithBloomError.class);
072
073  private static final Logger LOG =
074    LoggerFactory.getLogger(TestScanWithBloomError.class);
075
076  private static final String TABLE_NAME = "ScanWithBloomError";
077  private static final String FAMILY = "myCF";
078  private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
079  private static final String ROW = "theRow";
080  private static final String QUALIFIER_PREFIX = "qual";
081  private static final byte[] ROW_BYTES = Bytes.toBytes(ROW);
082  private static NavigableSet<Integer> allColIds = new TreeSet<>();
083  private HRegion region;
084  private BloomType bloomType;
085  private FileSystem fs;
086  private Configuration conf;
087
088  private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
089
090  @Parameters
091  public static final Collection<Object[]> parameters() {
092    List<Object[]> configurations = new ArrayList<>();
093    for (BloomType bloomType : BloomType.values()) {
094      configurations.add(new Object[] { bloomType });
095    }
096    return configurations;
097  }
098
099  public TestScanWithBloomError(BloomType bloomType) {
100    this.bloomType = bloomType;
101  }
102
103  @Before
104  public void setUp() throws IOException{
105    conf = TEST_UTIL.getConfiguration();
106    fs = FileSystem.get(conf);
107    conf.setInt(BloomFilterUtil.PREFIX_LENGTH_KEY, 10);
108  }
109
110  @Test
111  public void testThreeStoreFiles() throws IOException {
112    region = TEST_UTIL.createTestRegion(TABLE_NAME,
113        new HColumnDescriptor(FAMILY)
114            .setCompressionType(Compression.Algorithm.GZ)
115            .setBloomFilterType(bloomType)
116            .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS));
117    createStoreFile(new int[] {1, 2, 6});
118    createStoreFile(new int[] {1, 2, 3, 7});
119    createStoreFile(new int[] {1, 9});
120    scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7});
121
122    HBaseTestingUtility.closeRegionAndWAL(region);
123  }
124
125  private void scanColSet(int[] colSet, int[] expectedResultCols)
126      throws IOException {
127    LOG.info("Scanning column set: " + Arrays.toString(colSet));
128    Scan scan = new Scan(ROW_BYTES, ROW_BYTES);
129    addColumnSetToScan(scan, colSet);
130    RegionScannerImpl scanner = region.getScanner(scan);
131    KeyValueHeap storeHeap = scanner.getStoreHeapForTesting();
132    assertEquals(0, storeHeap.getHeap().size());
133    StoreScanner storeScanner =
134        (StoreScanner) storeHeap.getCurrentForTesting();
135    @SuppressWarnings({ "unchecked", "rawtypes" })
136    List<StoreFileScanner> scanners = (List<StoreFileScanner>)
137        (List) storeScanner.getAllScannersForTesting();
138
139    // Sort scanners by their HFile's modification time.
140    Collections.sort(scanners, new Comparator<StoreFileScanner>() {
141      @Override
142      public int compare(StoreFileScanner s1, StoreFileScanner s2) {
143        Path p1 = s1.getReader().getHFileReader().getPath();
144        Path p2 = s2.getReader().getHFileReader().getPath();
145        long t1, t2;
146        try {
147          t1 = fs.getFileStatus(p1).getModificationTime();
148          t2 = fs.getFileStatus(p2).getModificationTime();
149        } catch (IOException ex) {
150          throw new RuntimeException(ex);
151        }
152        return t1 < t2 ? -1 : t1 == t2 ? 1 : 0;
153      }
154    });
155
156    StoreFileReader lastStoreFileReader = null;
157    for (StoreFileScanner sfScanner : scanners)
158      lastStoreFileReader = sfScanner.getReader();
159
160    new HFilePrettyPrinter(conf).run(new String[]{ "-m", "-p", "-f",
161        lastStoreFileReader.getHFileReader().getPath().toString()});
162
163    // Disable Bloom filter for the last store file. The disabled Bloom filter
164    // will always return "true".
165    LOG.info("Disabling Bloom filter for: "
166        + lastStoreFileReader.getHFileReader().getName());
167    lastStoreFileReader.disableBloomFilterForTesting();
168
169    List<Cell> allResults = new ArrayList<>();
170
171    { // Limit the scope of results.
172      List<Cell> results = new ArrayList<>();
173      while (scanner.next(results) || results.size() > 0) {
174        allResults.addAll(results);
175        results.clear();
176      }
177    }
178
179    List<Integer> actualIds = new ArrayList<>();
180    for (Cell kv : allResults) {
181      String qual = Bytes.toString(CellUtil.cloneQualifier(kv));
182      assertTrue(qual.startsWith(QUALIFIER_PREFIX));
183      actualIds.add(Integer.valueOf(qual.substring(
184          QUALIFIER_PREFIX.length())));
185    }
186    List<Integer> expectedIds = new ArrayList<>();
187    for (int expectedId : expectedResultCols)
188      expectedIds.add(expectedId);
189
190    LOG.info("Column ids returned: " + actualIds + ", expected: "
191        + expectedIds);
192    assertEquals(expectedIds.toString(), actualIds.toString());
193  }
194
195  private void addColumnSetToScan(Scan scan, int[] colIds) {
196    for (int colId : colIds) {
197      scan.addColumn(FAMILY_BYTES,
198          Bytes.toBytes(qualFromId(colId)));
199    }
200  }
201
202  private String qualFromId(int colId) {
203    return QUALIFIER_PREFIX + colId;
204  }
205
206  private void createStoreFile(int[] colIds)
207      throws IOException {
208    Put p = new Put(ROW_BYTES);
209    for (int colId : colIds) {
210      long ts = Long.MAX_VALUE;
211      String qual = qualFromId(colId);
212      allColIds.add(colId);
213      KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY,
214          qual, ts, TestMultiColumnScanner.createValue(ROW, qual, ts));
215      p.add(kv);
216    }
217    region.put(p);
218    region.flush(true);
219  }
220
221
222}
223