001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Collection;
027import java.util.Collections;
028import java.util.Comparator;
029import java.util.List;
030import java.util.NavigableSet;
031import java.util.TreeSet;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.Cell;
036import org.apache.hadoop.hbase.CellUtil;
037import org.apache.hadoop.hbase.HBaseClassTestRule;
038import org.apache.hadoop.hbase.HBaseTestingUtility;
039import org.apache.hadoop.hbase.HColumnDescriptor;
040import org.apache.hadoop.hbase.KeyValue;
041import org.apache.hadoop.hbase.KeyValueTestUtil;
042import org.apache.hadoop.hbase.client.Put;
043import org.apache.hadoop.hbase.client.Scan;
044import org.apache.hadoop.hbase.io.compress.Compression;
045import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
046import org.apache.hadoop.hbase.testclassification.RegionServerTests;
047import org.apache.hadoop.hbase.testclassification.SmallTests;
048import org.apache.hadoop.hbase.util.BloomFilterUtil;
049import org.apache.hadoop.hbase.util.Bytes;
050import org.junit.Before;
051import org.junit.ClassRule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.runner.RunWith;
055import org.junit.runners.Parameterized;
056import org.junit.runners.Parameterized.Parameters;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059
060/**
061 * Test a multi-column scanner when there is a Bloom filter false-positive. This is needed for the
062 * multi-column Bloom filter optimization.
063 */
064@RunWith(Parameterized.class)
065@Category({ RegionServerTests.class, SmallTests.class })
066public class TestScanWithBloomError {
067
068  @ClassRule
069  public static final HBaseClassTestRule CLASS_RULE =
070    HBaseClassTestRule.forClass(TestScanWithBloomError.class);
071
072  private static final Logger LOG = LoggerFactory.getLogger(TestScanWithBloomError.class);
073
074  private static final String TABLE_NAME = "ScanWithBloomError";
075  private static final String FAMILY = "myCF";
076  private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
077  private static final String ROW = "theRow";
078  private static final String QUALIFIER_PREFIX = "qual";
079  private static final byte[] ROW_BYTES = Bytes.toBytes(ROW);
080  private static NavigableSet<Integer> allColIds = new TreeSet<>();
081  private HRegion region;
082  private BloomType bloomType;
083  private FileSystem fs;
084  private Configuration conf;
085
086  private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
087
088  @Parameters
089  public static final Collection<Object[]> parameters() {
090    List<Object[]> configurations = new ArrayList<>();
091    for (BloomType bloomType : BloomType.values()) {
092      configurations.add(new Object[] { bloomType });
093    }
094    return configurations;
095  }
096
097  public TestScanWithBloomError(BloomType bloomType) {
098    this.bloomType = bloomType;
099  }
100
101  @Before
102  public void setUp() throws IOException {
103    conf = TEST_UTIL.getConfiguration();
104    fs = FileSystem.get(conf);
105    conf.setInt(BloomFilterUtil.PREFIX_LENGTH_KEY, 10);
106  }
107
108  @Test
109  public void testThreeStoreFiles() throws IOException {
110    region = TEST_UTIL.createTestRegion(TABLE_NAME,
111      new HColumnDescriptor(FAMILY).setCompressionType(Compression.Algorithm.GZ)
112        .setBloomFilterType(bloomType).setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS));
113    createStoreFile(new int[] { 1, 2, 6 });
114    createStoreFile(new int[] { 1, 2, 3, 7 });
115    createStoreFile(new int[] { 1, 9 });
116    scanColSet(new int[] { 1, 4, 6, 7 }, new int[] { 1, 6, 7 });
117
118    HBaseTestingUtility.closeRegionAndWAL(region);
119  }
120
121  private void scanColSet(int[] colSet, int[] expectedResultCols) throws IOException {
122    LOG.info("Scanning column set: " + Arrays.toString(colSet));
123    Scan scan = new Scan(ROW_BYTES, ROW_BYTES);
124    addColumnSetToScan(scan, colSet);
125    RegionScannerImpl scanner = region.getScanner(scan);
126    KeyValueHeap storeHeap = scanner.storeHeap;
127    assertEquals(0, storeHeap.getHeap().size());
128    StoreScanner storeScanner = (StoreScanner) storeHeap.getCurrentForTesting();
129    @SuppressWarnings({ "unchecked", "rawtypes" })
130    List<StoreFileScanner> scanners =
131      (List<StoreFileScanner>) (List) storeScanner.getAllScannersForTesting();
132
133    // Sort scanners by their HFile's modification time.
134    Collections.sort(scanners, new Comparator<StoreFileScanner>() {
135      @Override
136      public int compare(StoreFileScanner s1, StoreFileScanner s2) {
137        Path p1 = s1.getReader().getHFileReader().getPath();
138        Path p2 = s2.getReader().getHFileReader().getPath();
139        long t1, t2;
140        try {
141          t1 = fs.getFileStatus(p1).getModificationTime();
142          t2 = fs.getFileStatus(p2).getModificationTime();
143        } catch (IOException ex) {
144          throw new RuntimeException(ex);
145        }
146        return t1 < t2 ? -1 : t1 == t2 ? 1 : 0;
147      }
148    });
149
150    StoreFileReader lastStoreFileReader = null;
151    for (StoreFileScanner sfScanner : scanners)
152      lastStoreFileReader = sfScanner.getReader();
153
154    new HFilePrettyPrinter(conf).run(
155      new String[] { "-m", "-p", "-f", lastStoreFileReader.getHFileReader().getPath().toString() });
156
157    // Disable Bloom filter for the last store file. The disabled Bloom filter
158    // will always return "true".
159    LOG.info("Disabling Bloom filter for: " + lastStoreFileReader.getHFileReader().getName());
160    lastStoreFileReader.disableBloomFilterForTesting();
161
162    List<Cell> allResults = new ArrayList<>();
163
164    { // Limit the scope of results.
165      List<Cell> results = new ArrayList<>();
166      while (scanner.next(results) || results.size() > 0) {
167        allResults.addAll(results);
168        results.clear();
169      }
170    }
171
172    List<Integer> actualIds = new ArrayList<>();
173    for (Cell kv : allResults) {
174      String qual = Bytes.toString(CellUtil.cloneQualifier(kv));
175      assertTrue(qual.startsWith(QUALIFIER_PREFIX));
176      actualIds.add(Integer.valueOf(qual.substring(QUALIFIER_PREFIX.length())));
177    }
178    List<Integer> expectedIds = new ArrayList<>();
179    for (int expectedId : expectedResultCols)
180      expectedIds.add(expectedId);
181
182    LOG.info("Column ids returned: " + actualIds + ", expected: " + expectedIds);
183    assertEquals(expectedIds.toString(), actualIds.toString());
184  }
185
186  private void addColumnSetToScan(Scan scan, int[] colIds) {
187    for (int colId : colIds) {
188      scan.addColumn(FAMILY_BYTES, Bytes.toBytes(qualFromId(colId)));
189    }
190  }
191
192  private String qualFromId(int colId) {
193    return QUALIFIER_PREFIX + colId;
194  }
195
196  private void createStoreFile(int[] colIds) throws IOException {
197    Put p = new Put(ROW_BYTES);
198    for (int colId : colIds) {
199      long ts = Long.MAX_VALUE;
200      String qual = qualFromId(colId);
201      allColIds.add(colId);
202      KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY, qual, ts,
203        TestMultiColumnScanner.createValue(ROW, qual, ts));
204      p.add(kv);
205    }
206    region.put(p);
207    region.flush(true);
208  }
209
210}