001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Collection;
027import java.util.Collections;
028import java.util.Comparator;
029import java.util.List;
030import java.util.NavigableSet;
031import java.util.TreeSet;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.Cell;
036import org.apache.hadoop.hbase.CellUtil;
037import org.apache.hadoop.hbase.HBaseClassTestRule;
038import org.apache.hadoop.hbase.HBaseTestingUtility;
039import org.apache.hadoop.hbase.HColumnDescriptor;
040import org.apache.hadoop.hbase.KeyValue;
041import org.apache.hadoop.hbase.KeyValueTestUtil;
042import org.apache.hadoop.hbase.client.Put;
043import org.apache.hadoop.hbase.client.Scan;
044import org.apache.hadoop.hbase.io.compress.Compression;
045import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
046import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
047import org.apache.hadoop.hbase.testclassification.RegionServerTests;
048import org.apache.hadoop.hbase.testclassification.SmallTests;
049import org.apache.hadoop.hbase.util.Bytes;
050import org.junit.Before;
051import org.junit.ClassRule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.runner.RunWith;
055import org.junit.runners.Parameterized;
056import org.junit.runners.Parameterized.Parameters;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059
060/**
061 * Test a multi-column scanner when there is a Bloom filter false-positive.
062 * This is needed for the multi-column Bloom filter optimization.
063 */
064@RunWith(Parameterized.class)
065@Category({RegionServerTests.class, SmallTests.class})
066public class TestScanWithBloomError {
067
068  @ClassRule
069  public static final HBaseClassTestRule CLASS_RULE =
070      HBaseClassTestRule.forClass(TestScanWithBloomError.class);
071
072  private static final Logger LOG =
073    LoggerFactory.getLogger(TestScanWithBloomError.class);
074
075  private static final String TABLE_NAME = "ScanWithBloomError";
076  private static final String FAMILY = "myCF";
077  private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
078  private static final String ROW = "theRow";
079  private static final String QUALIFIER_PREFIX = "qual";
080  private static final byte[] ROW_BYTES = Bytes.toBytes(ROW);
081  private static NavigableSet<Integer> allColIds = new TreeSet<>();
082  private HRegion region;
083  private BloomType bloomType;
084  private FileSystem fs;
085  private Configuration conf;
086
087  private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
088
089  @Parameters
090  public static final Collection<Object[]> parameters() {
091    List<Object[]> configurations = new ArrayList<>();
092    for (BloomType bloomType : BloomType.values()) {
093      configurations.add(new Object[] { bloomType });
094    }
095    return configurations;
096  }
097
098  public TestScanWithBloomError(BloomType bloomType) {
099    this.bloomType = bloomType;
100  }
101
102  @Before
103  public void setUp() throws IOException{
104    conf = TEST_UTIL.getConfiguration();
105    fs = FileSystem.get(conf);
106  }
107
108  @Test
109  public void testThreeStoreFiles() throws IOException {
110    region = TEST_UTIL.createTestRegion(TABLE_NAME,
111        new HColumnDescriptor(FAMILY)
112            .setCompressionType(Compression.Algorithm.GZ)
113            .setBloomFilterType(bloomType)
114            .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS));
115    createStoreFile(new int[] {1, 2, 6});
116    createStoreFile(new int[] {1, 2, 3, 7});
117    createStoreFile(new int[] {1, 9});
118    scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7});
119
120    HBaseTestingUtility.closeRegionAndWAL(region);
121  }
122
123  private void scanColSet(int[] colSet, int[] expectedResultCols)
124      throws IOException {
125    LOG.info("Scanning column set: " + Arrays.toString(colSet));
126    Scan scan = new Scan(ROW_BYTES, ROW_BYTES);
127    addColumnSetToScan(scan, colSet);
128    RegionScannerImpl scanner = region.getScanner(scan);
129    KeyValueHeap storeHeap = scanner.getStoreHeapForTesting();
130    assertEquals(0, storeHeap.getHeap().size());
131    StoreScanner storeScanner =
132        (StoreScanner) storeHeap.getCurrentForTesting();
133    @SuppressWarnings({ "unchecked", "rawtypes" })
134    List<StoreFileScanner> scanners = (List<StoreFileScanner>)
135        (List) storeScanner.getAllScannersForTesting();
136
137    // Sort scanners by their HFile's modification time.
138    Collections.sort(scanners, new Comparator<StoreFileScanner>() {
139      @Override
140      public int compare(StoreFileScanner s1, StoreFileScanner s2) {
141        Path p1 = s1.getReader().getHFileReader().getPath();
142        Path p2 = s2.getReader().getHFileReader().getPath();
143        long t1, t2;
144        try {
145          t1 = fs.getFileStatus(p1).getModificationTime();
146          t2 = fs.getFileStatus(p2).getModificationTime();
147        } catch (IOException ex) {
148          throw new RuntimeException(ex);
149        }
150        return t1 < t2 ? -1 : t1 == t2 ? 1 : 0;
151      }
152    });
153
154    StoreFileReader lastStoreFileReader = null;
155    for (StoreFileScanner sfScanner : scanners)
156      lastStoreFileReader = sfScanner.getReader();
157
158    new HFilePrettyPrinter(conf).run(new String[]{ "-m", "-p", "-f",
159        lastStoreFileReader.getHFileReader().getPath().toString()});
160
161    // Disable Bloom filter for the last store file. The disabled Bloom filter
162    // will always return "true".
163    LOG.info("Disabling Bloom filter for: "
164        + lastStoreFileReader.getHFileReader().getName());
165    lastStoreFileReader.disableBloomFilterForTesting();
166
167    List<Cell> allResults = new ArrayList<>();
168
169    { // Limit the scope of results.
170      List<Cell> results = new ArrayList<>();
171      while (scanner.next(results) || results.size() > 0) {
172        allResults.addAll(results);
173        results.clear();
174      }
175    }
176
177    List<Integer> actualIds = new ArrayList<>();
178    for (Cell kv : allResults) {
179      String qual = Bytes.toString(CellUtil.cloneQualifier(kv));
180      assertTrue(qual.startsWith(QUALIFIER_PREFIX));
181      actualIds.add(Integer.valueOf(qual.substring(
182          QUALIFIER_PREFIX.length())));
183    }
184    List<Integer> expectedIds = new ArrayList<>();
185    for (int expectedId : expectedResultCols)
186      expectedIds.add(expectedId);
187
188    LOG.info("Column ids returned: " + actualIds + ", expected: "
189        + expectedIds);
190    assertEquals(expectedIds.toString(), actualIds.toString());
191  }
192
193  private void addColumnSetToScan(Scan scan, int[] colIds) {
194    for (int colId : colIds) {
195      scan.addColumn(FAMILY_BYTES,
196          Bytes.toBytes(qualFromId(colId)));
197    }
198  }
199
200  private String qualFromId(int colId) {
201    return QUALIFIER_PREFIX + colId;
202  }
203
204  private void createStoreFile(int[] colIds)
205      throws IOException {
206    Put p = new Put(ROW_BYTES);
207    for (int colId : colIds) {
208      long ts = Long.MAX_VALUE;
209      String qual = qualFromId(colId);
210      allColIds.add(colId);
211      KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY,
212          qual, ts, TestMultiColumnScanner.createValue(ROW, qual, ts));
213      p.add(kv);
214    }
215    region.put(p);
216    region.flush(true);
217  }
218
219
220}
221