001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertArrayEquals;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertNull;
025import static org.junit.Assert.assertTrue;
026import static org.junit.Assert.fail;
027import static org.mockito.ArgumentMatchers.any;
028import static org.mockito.Mockito.mock;
029import static org.mockito.Mockito.when;
030
031import java.io.IOException;
032import java.nio.ByteBuffer;
033import java.util.ArrayList;
034import java.util.Arrays;
035import java.util.Collections;
036import java.util.Comparator;
037import java.util.List;
038import java.util.Map;
039import java.util.OptionalLong;
040import java.util.TreeSet;
041import java.util.concurrent.atomic.AtomicInteger;
042import org.apache.hadoop.conf.Configuration;
043import org.apache.hadoop.fs.FileSystem;
044import org.apache.hadoop.fs.Path;
045import org.apache.hadoop.hbase.Cell;
046import org.apache.hadoop.hbase.CellUtil;
047import org.apache.hadoop.hbase.HBaseClassTestRule;
048import org.apache.hadoop.hbase.HBaseTestingUtil;
049import org.apache.hadoop.hbase.HConstants;
050import org.apache.hadoop.hbase.KeyValue;
051import org.apache.hadoop.hbase.KeyValueUtil;
052import org.apache.hadoop.hbase.PrivateCellUtil;
053import org.apache.hadoop.hbase.TableDescriptors;
054import org.apache.hadoop.hbase.TableName;
055import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
056import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
057import org.apache.hadoop.hbase.client.RegionInfo;
058import org.apache.hadoop.hbase.client.RegionInfoBuilder;
059import org.apache.hadoop.hbase.client.Scan;
060import org.apache.hadoop.hbase.client.TableDescriptor;
061import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
062import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
063import org.apache.hadoop.hbase.io.HFileLink;
064import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
065import org.apache.hadoop.hbase.io.hfile.BlockCache;
066import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory;
067import org.apache.hadoop.hbase.io.hfile.CacheConfig;
068import org.apache.hadoop.hbase.io.hfile.CacheStats;
069import org.apache.hadoop.hbase.io.hfile.HFileContext;
070import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
071import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
072import org.apache.hadoop.hbase.io.hfile.HFileInfo;
073import org.apache.hadoop.hbase.io.hfile.HFileScanner;
074import org.apache.hadoop.hbase.io.hfile.ReaderContext;
075import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder;
076import org.apache.hadoop.hbase.master.MasterServices;
077import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
078import org.apache.hadoop.hbase.testclassification.MediumTests;
079import org.apache.hadoop.hbase.testclassification.RegionServerTests;
080import org.apache.hadoop.hbase.util.BloomFilterFactory;
081import org.apache.hadoop.hbase.util.Bytes;
082import org.apache.hadoop.hbase.util.ChecksumType;
083import org.apache.hadoop.hbase.util.CommonFSUtils;
084import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
085import org.junit.AfterClass;
086import org.junit.Before;
087import org.junit.ClassRule;
088import org.junit.Rule;
089import org.junit.Test;
090import org.junit.experimental.categories.Category;
091import org.junit.rules.TestName;
092import org.mockito.Mockito;
093import org.slf4j.Logger;
094import org.slf4j.LoggerFactory;
095
096import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
097import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
098import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
099
100/**
101 * Test HStoreFile
102 */
103@Category({ RegionServerTests.class, MediumTests.class })
104public class TestHStoreFile {
105
106  @ClassRule
107  public static final HBaseClassTestRule CLASS_RULE =
108    HBaseClassTestRule.forClass(TestHStoreFile.class);
109
110  private static final Logger LOG = LoggerFactory.getLogger(TestHStoreFile.class);
111  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
112  private CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
113  private static String ROOT_DIR = TEST_UTIL.getDataTestDir("TestStoreFile").toString();
114  private static final ChecksumType CKTYPE = ChecksumType.CRC32C;
115  private static final int CKBYTES = 512;
116  private static String TEST_FAMILY = "cf";
117  private static final char FIRST_CHAR = 'a';
118  private static final char LAST_CHAR = 'z';
119
120  @Rule
121  public TestName name = new TestName();
122
123  private Configuration conf;
124  private Path testDir;
125  private FileSystem fs;
126
127  @Before
128  public void setUp() throws IOException {
129    conf = TEST_UTIL.getConfiguration();
130    testDir = TEST_UTIL.getDataTestDir(name.getMethodName());
131    fs = testDir.getFileSystem(conf);
132  }
133
134  @AfterClass
135  public static void tearDownAfterClass() {
136    TEST_UTIL.cleanupTestDir();
137  }
138
139  /**
140   * Write a file and then assert that we can read from top and bottom halves using two
141   * HalfMapFiles, as well as one HalfMapFile and one HFileLink file.
142   */
143  @Test
144  public void testBasicHalfAndHFileLinkMapFile() throws Exception {
145    final RegionInfo hri =
146      RegionInfoBuilder.newBuilder(TableName.valueOf("testBasicHalfAndHFileLinkMapFile")).build();
147    // The locations of HFileLink refers hfiles only should be consistent with the table dir
148    // create by CommonFSUtils directory, so we should make the region directory under
149    // the mode of CommonFSUtils.getTableDir here.
150    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
151      CommonFSUtils.getTableDir(CommonFSUtils.getRootDir(conf), hri.getTable()), hri);
152
153    HFileContext meta = new HFileContextBuilder().withBlockSize(2 * 1024).build();
154    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
155      .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
156    writeStoreFile(writer);
157
158    Path sfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
159    HStoreFile sf = new HStoreFile(this.fs, sfPath, conf, cacheConf, BloomType.NONE, true);
160    checkHalfHFile(regionFs, sf);
161  }
162
163  private void writeStoreFile(final StoreFileWriter writer) throws IOException {
164    writeStoreFile(writer, Bytes.toBytes(name.getMethodName()),
165      Bytes.toBytes(name.getMethodName()));
166  }
167
168  // pick an split point (roughly halfway)
169  byte[] SPLITKEY = new byte[] { (LAST_CHAR + FIRST_CHAR) / 2, FIRST_CHAR };
170
171  /*
172   * Writes HStoreKey and ImmutableBytes data to passed writer and then closes it. nn
173   */
174  public static void writeStoreFile(final StoreFileWriter writer, byte[] fam, byte[] qualifier)
175    throws IOException {
176    long now = EnvironmentEdgeManager.currentTime();
177    try {
178      for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) {
179        for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) {
180          byte[] b = new byte[] { (byte) d, (byte) e };
181          writer.append(new KeyValue(b, fam, qualifier, now, b));
182        }
183      }
184    } finally {
185      writer.close();
186    }
187  }
188
189  /**
190   * Test that our mechanism of writing store files in one region to reference store files in other
191   * regions works.
192   */
193  @Test
194  public void testReference() throws IOException {
195    final RegionInfo hri =
196      RegionInfoBuilder.newBuilder(TableName.valueOf("testReferenceTb")).build();
197    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
198      new Path(testDir, hri.getTable().getNameAsString()), hri);
199
200    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
201    // Make a store file and write data to it.
202    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
203      .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
204    writeStoreFile(writer);
205
206    Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
207    HStoreFile hsf = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true);
208    hsf.initReader();
209    StoreFileReader reader = hsf.getReader();
210    // Split on a row, not in middle of row. Midkey returned by reader
211    // may be in middle of row. Create new one with empty column and
212    // timestamp.
213    byte[] midRow = CellUtil.cloneRow(reader.midKey().get());
214    byte[] finalRow = CellUtil.cloneRow(reader.getLastKey().get());
215    hsf.closeStoreFile(true);
216
217    // Make a reference
218    RegionInfo splitHri = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(midRow).build();
219    Path refPath = splitStoreFile(regionFs, splitHri, TEST_FAMILY, hsf, midRow, true);
220    HStoreFile refHsf = new HStoreFile(this.fs, refPath, conf, cacheConf, BloomType.NONE, true);
221    refHsf.initReader();
222    // Now confirm that I can read from the reference and that it only gets
223    // keys from top half of the file.
224    HFileScanner s = refHsf.getReader().getScanner(false, false);
225    Cell kv = null;
226    for (boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) {
227      ByteBuffer bb = ByteBuffer.wrap(((KeyValue) s.getKey()).getKey());
228      kv = KeyValueUtil.createKeyValueFromKey(bb);
229      if (first) {
230        assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), midRow, 0,
231          midRow.length));
232        first = false;
233      }
234    }
235    assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), finalRow, 0,
236      finalRow.length));
237  }
238
239  @Test
240  public void testStoreFileReference() throws Exception {
241    final RegionInfo hri =
242      RegionInfoBuilder.newBuilder(TableName.valueOf("testStoreFileReference")).build();
243    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
244      new Path(testDir, hri.getTable().getNameAsString()), hri);
245    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
246
247    // Make a store file and write data to it.
248    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
249      .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
250    writeStoreFile(writer);
251    Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
252    writer.close();
253
254    HStoreFile file = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true);
255    file.initReader();
256    StoreFileReader r = file.getReader();
257    assertNotNull(r);
258    StoreFileScanner scanner =
259      new StoreFileScanner(r, mock(HFileScanner.class), false, false, 0, 0, false);
260
261    // Verify after instantiating scanner refCount is increased
262    assertTrue("Verify file is being referenced", file.isReferencedInReads());
263    scanner.close();
264    // Verify after closing scanner refCount is decreased
265    assertFalse("Verify file is not being referenced", file.isReferencedInReads());
266  }
267
268  @Test
269  public void testEmptyStoreFileRestrictKeyRanges() throws Exception {
270    StoreFileReader reader = mock(StoreFileReader.class);
271    HStore store = mock(HStore.class);
272    byte[] cf = Bytes.toBytes("ty");
273    ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.of(cf);
274    when(store.getColumnFamilyDescriptor()).thenReturn(cfd);
275    try (StoreFileScanner scanner =
276      new StoreFileScanner(reader, mock(HFileScanner.class), false, false, 0, 0, true)) {
277      Scan scan = new Scan();
278      scan.setColumnFamilyTimeRange(cf, 0, 1);
279      assertFalse(scanner.shouldUseScanner(scan, store, 0));
280    }
281  }
282
283  @Test
284  public void testHFileLink() throws IOException {
285    final RegionInfo hri =
286      RegionInfoBuilder.newBuilder(TableName.valueOf("testHFileLinkTb")).build();
287    // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
288    Configuration testConf = new Configuration(this.conf);
289    CommonFSUtils.setRootDir(testConf, testDir);
290    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
291      CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);
292    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
293
294    // Make a store file and write data to it.
295    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
296      .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
297    writeStoreFile(writer);
298
299    Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
300    Path dstPath = new Path(regionFs.getTableDir(), new Path("test-region", TEST_FAMILY));
301    HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
302    Path linkFilePath =
303      new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
304
305    // Try to open store file from link
306    StoreFileInfo storeFileInfo = new StoreFileInfo(testConf, this.fs, linkFilePath, true);
307    HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf);
308    assertTrue(storeFileInfo.isLink());
309    hsf.initReader();
310
311    // Now confirm that I can read from the link
312    int count = 1;
313    HFileScanner s = hsf.getReader().getScanner(false, false);
314    s.seekTo();
315    while (s.next()) {
316      count++;
317    }
318    assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
319  }
320
321  /**
322   * This test creates an hfile and then the dir structures and files to verify that references to
323   * hfilelinks (created by snapshot clones) can be properly interpreted.
324   */
325  @Test
326  public void testReferenceToHFileLink() throws IOException {
327    // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
328    Configuration testConf = new Configuration(this.conf);
329    CommonFSUtils.setRootDir(testConf, testDir);
330
331    // adding legal table name chars to verify regex handles it.
332    RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("_original-evil-name")).build();
333    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
334      CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);
335
336    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
337    // Make a store file and write data to it. <root>/<tablename>/<rgn>/<cf>/<file>
338    StoreFileWriter writer = new StoreFileWriter.Builder(testConf, cacheConf, this.fs)
339      .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
340    writeStoreFile(writer);
341    Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
342
343    // create link to store file. <root>/clone/region/<cf>/<hfile>-<region>-<table>
344    RegionInfo hriClone = RegionInfoBuilder.newBuilder(TableName.valueOf("clone")).build();
345    HRegionFileSystem cloneRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
346      CommonFSUtils.getTableDir(testDir, hri.getTable()), hriClone);
347    Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY);
348    HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
349    Path linkFilePath =
350      new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
351
352    // create splits of the link.
353    // <root>/clone/splitA/<cf>/<reftohfilelink>,
354    // <root>/clone/splitB/<cf>/<reftohfilelink>
355    RegionInfo splitHriA = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(SPLITKEY).build();
356    RegionInfo splitHriB =
357      RegionInfoBuilder.newBuilder(hri.getTable()).setStartKey(SPLITKEY).build();
358    HStoreFile f = new HStoreFile(fs, linkFilePath, testConf, cacheConf, BloomType.NONE, true);
359    f.initReader();
360    Path pathA = splitStoreFile(cloneRegionFs, splitHriA, TEST_FAMILY, f, SPLITKEY, true); // top
361    Path pathB = splitStoreFile(cloneRegionFs, splitHriB, TEST_FAMILY, f, SPLITKEY, false);// bottom
362    f.closeStoreFile(true);
363    // OK test the thing
364    CommonFSUtils.logFileSystemState(fs, testDir, LOG);
365
366    // There is a case where a file with the hfilelink pattern is actually a daughter
367    // reference to a hfile link. This code in StoreFile that handles this case.
368
369    // Try to open store file from link
370    HStoreFile hsfA = new HStoreFile(this.fs, pathA, testConf, cacheConf, BloomType.NONE, true);
371    hsfA.initReader();
372
373    // Now confirm that I can read from the ref to link
374    int count = 1;
375    HFileScanner s = hsfA.getReader().getScanner(false, false);
376    s.seekTo();
377    while (s.next()) {
378      count++;
379    }
380    assertTrue(count > 0); // read some rows here
381
382    // Try to open store file from link
383    HStoreFile hsfB = new HStoreFile(this.fs, pathB, testConf, cacheConf, BloomType.NONE, true);
384    hsfB.initReader();
385
386    // Now confirm that I can read from the ref to link
387    HFileScanner sB = hsfB.getReader().getScanner(false, false);
388    sB.seekTo();
389
390    // count++ as seekTo() will advance the scanner
391    count++;
392    while (sB.next()) {
393      count++;
394    }
395
396    // read the rest of the rows
397    assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
398  }
399
400  private void checkHalfHFile(final HRegionFileSystem regionFs, final HStoreFile f)
401    throws IOException {
402    f.initReader();
403    Cell midkey = f.getReader().midKey().get();
404    KeyValue midKV = (KeyValue) midkey;
405    // 1. test using the midRow as the splitKey, this test will generate two Reference files
406    // in the children
407    byte[] midRow = CellUtil.cloneRow(midKV);
408    // Create top split.
409    RegionInfo topHri =
410      RegionInfoBuilder.newBuilder(regionFs.getRegionInfo().getTable()).setEndKey(SPLITKEY).build();
411    Path topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, midRow, true);
412    // Create bottom split.
413    RegionInfo bottomHri = RegionInfoBuilder.newBuilder(regionFs.getRegionInfo().getTable())
414      .setStartKey(SPLITKEY).build();
415    Path bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, midRow, false);
416    // Make readers on top and bottom.
417    HStoreFile topF = new HStoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, true);
418    topF.initReader();
419    StoreFileReader top = topF.getReader();
420    HStoreFile bottomF = new HStoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE, true);
421    bottomF.initReader();
422    StoreFileReader bottom = bottomF.getReader();
423    ByteBuffer previous = null;
424    LOG.info("Midkey: " + midKV.toString());
425    ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midKV.getKey());
426    try {
427      // Now make two HalfMapFiles and assert they can read the full backing
428      // file, one from the top and the other from the bottom.
429      // Test bottom half first.
430      // Now test reading from the top.
431      boolean first = true;
432      ByteBuffer key = null;
433      HFileScanner topScanner = top.getScanner(false, false);
434      while (
435        (!topScanner.isSeeked() && topScanner.seekTo())
436          || (topScanner.isSeeked() && topScanner.next())
437      ) {
438        key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey());
439
440        if (
441          (PrivateCellUtil.compare(topScanner.getReader().getComparator(), midKV, key.array(),
442            key.arrayOffset(), key.limit())) > 0
443        ) {
444          fail("key=" + Bytes.toStringBinary(key) + " < midkey=" + midkey);
445        }
446        if (first) {
447          first = false;
448          LOG.info("First in top: " + Bytes.toString(Bytes.toBytes(key)));
449        }
450      }
451      LOG.info("Last in top: " + Bytes.toString(Bytes.toBytes(key)));
452
453      first = true;
454      HFileScanner bottomScanner = bottom.getScanner(false, false);
455      while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) {
456        previous = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
457        key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
458        if (first) {
459          first = false;
460          LOG.info("First in bottom: " + Bytes.toString(Bytes.toBytes(previous)));
461        }
462        assertTrue(key.compareTo(bbMidkeyBytes) < 0);
463      }
464      if (previous != null) {
465        LOG.info("Last in bottom: " + Bytes.toString(Bytes.toBytes(previous)));
466      }
467      // Remove references.
468      regionFs.cleanupDaughterRegion(topHri);
469      regionFs.cleanupDaughterRegion(bottomHri);
470
471      // 2. test using a midkey which will generate one Reference file and one HFileLink file.
472      // First, do a key that is < than first key. Ensure splits behave
473      // properly.
474      byte[] badmidkey = Bytes.toBytes("  .");
475      assertTrue(fs.exists(f.getPath()));
476      topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true);
477      bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false);
478
479      assertNull(bottomPath);
480
481      topF = new HStoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, true);
482      topF.initReader();
483      top = topF.getReader();
484      // Now read from the top.
485      first = true;
486      topScanner = top.getScanner(false, false);
487      KeyValue.KeyOnlyKeyValue keyOnlyKV = new KeyValue.KeyOnlyKeyValue();
488      while ((!topScanner.isSeeked() && topScanner.seekTo()) || topScanner.next()) {
489        key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey());
490        keyOnlyKV.setKey(key.array(), 0 + key.arrayOffset(), key.limit());
491        assertTrue(PrivateCellUtil.compare(topScanner.getReader().getComparator(), keyOnlyKV,
492          badmidkey, 0, badmidkey.length) >= 0);
493        if (first) {
494          first = false;
495          KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);
496          LOG.info("First top when key < bottom: " + keyKV);
497          String tmp =
498            Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
499          for (int i = 0; i < tmp.length(); i++) {
500            assertTrue(tmp.charAt(i) == 'a');
501          }
502        }
503      }
504      KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);
505      LOG.info("Last top when key < bottom: " + keyKV);
506      String tmp = Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
507      for (int i = 0; i < tmp.length(); i++) {
508        assertTrue(tmp.charAt(i) == 'z');
509      }
510      // Remove references.
511      regionFs.cleanupDaughterRegion(topHri);
512      regionFs.cleanupDaughterRegion(bottomHri);
513
514      // Test when badkey is > than last key in file ('||' > 'zz').
515      badmidkey = Bytes.toBytes("|||");
516      topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true);
517      bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false);
518      assertNull(topPath);
519
520      bottomF = new HStoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE, true);
521      bottomF.initReader();
522      bottom = bottomF.getReader();
523      first = true;
524      bottomScanner = bottom.getScanner(false, false);
525      while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) {
526        key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
527        if (first) {
528          first = false;
529          keyKV = KeyValueUtil.createKeyValueFromKey(key);
530          LOG.info("First bottom when key > top: " + keyKV);
531          tmp = Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
532          for (int i = 0; i < tmp.length(); i++) {
533            assertTrue(tmp.charAt(i) == 'a');
534          }
535        }
536      }
537      keyKV = KeyValueUtil.createKeyValueFromKey(key);
538      LOG.info("Last bottom when key > top: " + keyKV);
539      for (int i = 0; i < tmp.length(); i++) {
540        assertTrue(
541          Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength()).charAt(i)
542              == 'z');
543      }
544    } finally {
545      if (top != null) {
546        top.close(true); // evict since we are about to delete the file
547      }
548      if (bottom != null) {
549        bottom.close(true); // evict since we are about to delete the file
550      }
551      fs.delete(f.getPath(), true);
552    }
553  }
554
555  private static StoreFileScanner getStoreFileScanner(StoreFileReader reader, boolean cacheBlocks,
556    boolean pread) {
557    return reader.getStoreFileScanner(cacheBlocks, pread, false, 0, 0, false);
558  }
559
560  private static final String localFormatter = "%010d";
561
562  private void bloomWriteRead(StoreFileWriter writer, FileSystem fs) throws Exception {
563    float err = conf.getFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, 0);
564    Path f = writer.getPath();
565    long now = EnvironmentEdgeManager.currentTime();
566    for (int i = 0; i < 2000; i += 2) {
567      String row = String.format(localFormatter, i);
568      KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("col"),
569        now, Bytes.toBytes("value"));
570      writer.append(kv);
571    }
572    writer.close();
573
574    ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();
575    HFileInfo fileInfo = new HFileInfo(context, conf);
576    StoreFileReader reader =
577      new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf);
578    fileInfo.initMetaAndIndex(reader.getHFileReader());
579    reader.loadFileInfo();
580    reader.loadBloomfilter();
581    StoreFileScanner scanner = getStoreFileScanner(reader, false, false);
582
583    // check false positives rate
584    int falsePos = 0;
585    int falseNeg = 0;
586    for (int i = 0; i < 2000; i++) {
587      String row = String.format(localFormatter, i);
588      TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);
589      columns.add(Bytes.toBytes("family:col"));
590
591      Scan scan = new Scan().withStartRow(Bytes.toBytes(row)).withStopRow(Bytes.toBytes(row), true);
592      scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes("family:col"));
593      HStore store = mock(HStore.class);
594      when(store.getColumnFamilyDescriptor())
595        .thenReturn(ColumnFamilyDescriptorBuilder.of("family"));
596      boolean exists = scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);
597      if (i % 2 == 0) {
598        if (!exists) {
599          falseNeg++;
600        }
601      } else {
602        if (exists) {
603          falsePos++;
604        }
605      }
606    }
607    reader.close(true); // evict because we are about to delete the file
608    fs.delete(f, true);
609    assertEquals("False negatives: " + falseNeg, 0, falseNeg);
610    int maxFalsePos = (int) (2 * 2000 * err);
611    assertTrue("Too many false positives: " + falsePos + " (err=" + err + ", expected no more than "
612      + maxFalsePos + ")", falsePos <= maxFalsePos);
613  }
614
615  private static final int BLOCKSIZE_SMALL = 8192;
616
617  @Test
618  public void testBloomFilter() throws Exception {
619    conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01);
620    conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
621
622    // write the file
623    Path f = new Path(ROOT_DIR, name.getMethodName());
624    HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL)
625      .withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build();
626    // Make a store file and write data to it.
627    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)
628      .withBloomType(BloomType.ROW).withMaxKeyCount(2000).withFileContext(meta).build();
629    bloomWriteRead(writer, fs);
630  }
631
632  @Test
633  public void testDeleteFamilyBloomFilter() throws Exception {
634    conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01);
635    conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
636    float err = conf.getFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, 0);
637
638    // write the file
639    Path f = new Path(ROOT_DIR, name.getMethodName());
640
641    HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL)
642      .withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build();
643    // Make a store file and write data to it.
644    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)
645      .withMaxKeyCount(2000).withFileContext(meta).build();
646
647    // add delete family
648    long now = EnvironmentEdgeManager.currentTime();
649    for (int i = 0; i < 2000; i += 2) {
650      String row = String.format(localFormatter, i);
651      KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("col"),
652        now, KeyValue.Type.DeleteFamily, Bytes.toBytes("value"));
653      writer.append(kv);
654    }
655    writer.close();
656
657    ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();
658    HFileInfo fileInfo = new HFileInfo(context, conf);
659    StoreFileReader reader =
660      new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf);
661    fileInfo.initMetaAndIndex(reader.getHFileReader());
662    reader.loadFileInfo();
663    reader.loadBloomfilter();
664
665    // check false positives rate
666    int falsePos = 0;
667    int falseNeg = 0;
668    for (int i = 0; i < 2000; i++) {
669      String row = String.format(localFormatter, i);
670      byte[] rowKey = Bytes.toBytes(row);
671      boolean exists = reader.passesDeleteFamilyBloomFilter(rowKey, 0, rowKey.length);
672      if (i % 2 == 0) {
673        if (!exists) {
674          falseNeg++;
675        }
676      } else {
677        if (exists) {
678          falsePos++;
679        }
680      }
681    }
682    assertEquals(1000, reader.getDeleteFamilyCnt());
683    reader.close(true); // evict because we are about to delete the file
684    fs.delete(f, true);
685    assertEquals("False negatives: " + falseNeg, 0, falseNeg);
686    int maxFalsePos = (int) (2 * 2000 * err);
687    assertTrue("Too many false positives: " + falsePos + " (err=" + err + ", expected no more than "
688      + maxFalsePos, falsePos <= maxFalsePos);
689  }
690
691  /**
692   * Test for HBASE-8012
693   */
694  @Test
695  public void testReseek() throws Exception {
696    // write the file
697    Path f = new Path(ROOT_DIR, name.getMethodName());
698    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
699    // Make a store file and write data to it.
700    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)
701      .withFileContext(meta).build();
702
703    writeStoreFile(writer);
704    writer.close();
705
706    ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();
707    HFileInfo fileInfo = new HFileInfo(context, conf);
708    StoreFileReader reader =
709      new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf);
710    fileInfo.initMetaAndIndex(reader.getHFileReader());
711
712    // Now do reseek with empty KV to position to the beginning of the file
713
714    KeyValue k = KeyValueUtil.createFirstOnRow(HConstants.EMPTY_BYTE_ARRAY);
715    StoreFileScanner s = getStoreFileScanner(reader, false, false);
716    s.reseek(k);
717
718    assertNotNull("Intial reseek should position at the beginning of the file", s.peek());
719  }
720
721  @Test
722  public void testBloomTypes() throws Exception {
723    float err = (float) 0.01;
724    FileSystem fs = FileSystem.getLocal(conf);
725    conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, err);
726    conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
727
728    int rowCount = 50;
729    int colCount = 10;
730    int versions = 2;
731
732    // run once using columns and once using rows
733    BloomType[] bt = { BloomType.ROWCOL, BloomType.ROW };
734    int[] expKeys = { rowCount * colCount, rowCount };
735    // below line deserves commentary. it is expected bloom false positives
736    // column = rowCount*2*colCount inserts
737    // row-level = only rowCount*2 inserts, but failures will be magnified by
738    // 2nd for loop for every column (2*colCount)
739    float[] expErr = { 2 * rowCount * colCount * err, 2 * rowCount * 2 * colCount * err };
740
741    for (int x : new int[] { 0, 1 }) {
742      // write the file
743      Path f = new Path(ROOT_DIR, name.getMethodName() + x);
744      HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL)
745        .withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build();
746      // Make a store file and write data to it.
747      StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)
748        .withBloomType(bt[x]).withMaxKeyCount(expKeys[x]).withFileContext(meta).build();
749
750      long now = EnvironmentEdgeManager.currentTime();
751      for (int i = 0; i < rowCount * 2; i += 2) { // rows
752        for (int j = 0; j < colCount * 2; j += 2) { // column qualifiers
753          String row = String.format(localFormatter, i);
754          String col = String.format(localFormatter, j);
755          for (int k = 0; k < versions; ++k) { // versions
756            KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
757              Bytes.toBytes("col" + col), now - k, Bytes.toBytes(-1L));
758            writer.append(kv);
759          }
760        }
761      }
762      writer.close();
763
764      ReaderContext context =
765        new ReaderContextBuilder().withFilePath(f).withFileSize(fs.getFileStatus(f).getLen())
766          .withFileSystem(fs).withInputStreamWrapper(new FSDataInputStreamWrapper(fs, f)).build();
767      HFileInfo fileInfo = new HFileInfo(context, conf);
768      StoreFileReader reader =
769        new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf);
770      fileInfo.initMetaAndIndex(reader.getHFileReader());
771      reader.loadFileInfo();
772      reader.loadBloomfilter();
773      StoreFileScanner scanner = getStoreFileScanner(reader, false, false);
774      assertEquals(expKeys[x], reader.getGeneralBloomFilter().getKeyCount());
775
776      HStore store = mock(HStore.class);
777      when(store.getColumnFamilyDescriptor())
778        .thenReturn(ColumnFamilyDescriptorBuilder.of("family"));
779      // check false positives rate
780      int falsePos = 0;
781      int falseNeg = 0;
782      for (int i = 0; i < rowCount * 2; ++i) { // rows
783        for (int j = 0; j < colCount * 2; ++j) { // column qualifiers
784          String row = String.format(localFormatter, i);
785          String col = String.format(localFormatter, j);
786          TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);
787          columns.add(Bytes.toBytes("col" + col));
788
789          Scan scan =
790            new Scan().withStartRow(Bytes.toBytes(row)).withStopRow(Bytes.toBytes(row), true);
791          scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes(("col" + col)));
792
793          boolean exists = scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);
794          boolean shouldRowExist = i % 2 == 0;
795          boolean shouldColExist = j % 2 == 0;
796          shouldColExist = shouldColExist || bt[x] == BloomType.ROW;
797          if (shouldRowExist && shouldColExist) {
798            if (!exists) {
799              falseNeg++;
800            }
801          } else {
802            if (exists) {
803              falsePos++;
804            }
805          }
806        }
807      }
808      reader.close(true); // evict because we are about to delete the file
809      fs.delete(f, true);
810      System.out.println(bt[x].toString());
811      System.out.println("  False negatives: " + falseNeg);
812      System.out.println("  False positives: " + falsePos);
813      assertEquals(0, falseNeg);
814      assertTrue(falsePos < 2 * expErr[x]);
815    }
816  }
817
818  @Test
819  public void testSeqIdComparator() {
820    assertOrdering(StoreFileComparators.SEQ_ID, mockStoreFile(true, 100, 1000, -1, "/foo/123"),
821      mockStoreFile(true, 100, 1000, -1, "/foo/124"), mockStoreFile(true, 99, 1000, -1, "/foo/126"),
822      mockStoreFile(true, 98, 2000, -1, "/foo/126"), mockStoreFile(false, 3453, -1, 1, "/foo/1"),
823      mockStoreFile(false, 2, -1, 3, "/foo/2"), mockStoreFile(false, 1000, -1, 5, "/foo/2"),
824      mockStoreFile(false, 76, -1, 5, "/foo/3"));
825  }
826
827  /**
828   * Assert that the given comparator orders the given storefiles in the same way that they're
829   * passed.
830   */
831  private void assertOrdering(Comparator<? super HStoreFile> comparator, HStoreFile... sfs) {
832    ArrayList<HStoreFile> sorted = Lists.newArrayList(sfs);
833    Collections.shuffle(sorted);
834    Collections.sort(sorted, comparator);
835    LOG.debug("sfs: " + Joiner.on(",").join(sfs));
836    LOG.debug("sorted: " + Joiner.on(",").join(sorted));
837    assertTrue(Iterables.elementsEqual(Arrays.asList(sfs), sorted));
838  }
839
840  /**
841   * Create a mock StoreFile with the given attributes.
842   */
843  private HStoreFile mockStoreFile(boolean bulkLoad, long size, long bulkTimestamp, long seqId,
844    String path) {
845    HStoreFile mock = Mockito.mock(HStoreFile.class);
846    StoreFileReader reader = Mockito.mock(StoreFileReader.class);
847
848    Mockito.doReturn(size).when(reader).length();
849
850    Mockito.doReturn(reader).when(mock).getReader();
851    Mockito.doReturn(bulkLoad).when(mock).isBulkLoadResult();
852    Mockito.doReturn(OptionalLong.of(bulkTimestamp)).when(mock).getBulkLoadTimestamp();
853    Mockito.doReturn(seqId).when(mock).getMaxSequenceId();
854    Mockito.doReturn(new Path(path)).when(mock).getPath();
855    String name = "mock storefile, bulkLoad=" + bulkLoad + " bulkTimestamp=" + bulkTimestamp
856      + " seqId=" + seqId + " path=" + path;
857    Mockito.doReturn(name).when(mock).toString();
858    return mock;
859  }
860
861  /**
862   * Generate a list of KeyValues for testing based on given parameters
863   * @return the rows key-value list
864   */
865  List<KeyValue> getKeyValueSet(long[] timestamps, int numRows, byte[] qualifier, byte[] family) {
866    List<KeyValue> kvList = new ArrayList<>();
867    for (int i = 1; i <= numRows; i++) {
868      byte[] b = Bytes.toBytes(i);
869      LOG.info(Bytes.toString(b));
870      LOG.info(Bytes.toString(b));
871      for (long timestamp : timestamps) {
872        kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
873      }
874    }
875    return kvList;
876  }
877
878  /**
879   * Test to ensure correctness when using StoreFile with multiple timestamps
880   */
881  @Test
882  public void testMultipleTimestamps() throws IOException {
883    byte[] family = Bytes.toBytes("familyname");
884    byte[] qualifier = Bytes.toBytes("qualifier");
885    int numRows = 10;
886    long[] timestamps = new long[] { 20, 10, 5, 1 };
887    Scan scan = new Scan();
888
889    // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.
890    Path storedir = new Path(new Path(testDir, "7e0102"), Bytes.toString(family));
891    Path dir = new Path(storedir, "1234567890");
892    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
893    // Make a store file and write data to it.
894    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
895      .withOutputDir(dir).withFileContext(meta).build();
896
897    List<KeyValue> kvList = getKeyValueSet(timestamps, numRows, qualifier, family);
898
899    for (KeyValue kv : kvList) {
900      writer.append(kv);
901    }
902    writer.appendMetadata(0, false);
903    writer.close();
904
905    HStoreFile hsf =
906      new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);
907    HStore store = mock(HStore.class);
908    when(store.getColumnFamilyDescriptor()).thenReturn(ColumnFamilyDescriptorBuilder.of(family));
909    hsf.initReader();
910    StoreFileReader reader = hsf.getReader();
911    StoreFileScanner scanner = getStoreFileScanner(reader, false, false);
912    TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);
913    columns.add(qualifier);
914
915    scan.setTimeRange(20, 100);
916    assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));
917
918    scan.setTimeRange(1, 2);
919    assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));
920
921    scan.setTimeRange(8, 10);
922    assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));
923
924    // lets make sure it still works with column family time ranges
925    scan.setColumnFamilyTimeRange(family, 7, 50);
926    assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));
927
928    // This test relies on the timestamp range optimization
929    scan = new Scan();
930    scan.setTimeRange(27, 50);
931    assertTrue(!scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));
932
933    // should still use the scanner because we override the family time range
934    scan = new Scan();
935    scan.setTimeRange(27, 50);
936    scan.setColumnFamilyTimeRange(family, 7, 50);
937    assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));
938  }
939
940  @Test
941  public void testCacheOnWriteEvictOnClose() throws Exception {
942    Configuration conf = this.conf;
943
944    // Find a home for our files (regiondir ("7e0102") and familyname).
945    Path baseDir = new Path(new Path(testDir, "7e0102"), "twoCOWEOC");
946
947    // Grab the block cache and get the initial hit/miss counts
948    BlockCache bc = BlockCacheFactory.createBlockCache(conf);
949    assertNotNull(bc);
950    CacheStats cs = bc.getStats();
951    long startHit = cs.getHitCount();
952    long startMiss = cs.getMissCount();
953    long startEvicted = cs.getEvictedCount();
954
955    // Let's write a StoreFile with three blocks, with cache on write off
956    conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);
957    CacheConfig cacheConf = new CacheConfig(conf, bc);
958    Path pathCowOff = new Path(baseDir, "123456789");
959    StoreFileWriter writer = writeStoreFile(conf, cacheConf, pathCowOff, 3);
960    HStoreFile hsf =
961      new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);
962    LOG.debug(hsf.getPath().toString());
963
964    // Read this file, we should see 3 misses
965    hsf.initReader();
966    StoreFileReader reader = hsf.getReader();
967    reader.loadFileInfo();
968    StoreFileScanner scanner = getStoreFileScanner(reader, true, true);
969    scanner.seek(KeyValue.LOWESTKEY);
970    while (scanner.next() != null) {
971      continue;
972    }
973    assertEquals(startHit, cs.getHitCount());
974    assertEquals(startMiss + 3, cs.getMissCount());
975    assertEquals(startEvicted, cs.getEvictedCount());
976    startMiss += 3;
977    scanner.close();
978    reader.close(cacheConf.shouldEvictOnClose());
979
980    // Now write a StoreFile with three blocks, with cache on write on
981    conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true);
982    cacheConf = new CacheConfig(conf, bc);
983    Path pathCowOn = new Path(baseDir, "123456788");
984    writer = writeStoreFile(conf, cacheConf, pathCowOn, 3);
985    hsf = new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);
986
987    // Read this file, we should see 3 hits
988    hsf.initReader();
989    reader = hsf.getReader();
990    scanner = getStoreFileScanner(reader, true, true);
991    scanner.seek(KeyValue.LOWESTKEY);
992    while (scanner.next() != null) {
993      continue;
994    }
995    assertEquals(startHit + 3, cs.getHitCount());
996    assertEquals(startMiss, cs.getMissCount());
997    assertEquals(startEvicted, cs.getEvictedCount());
998    startHit += 3;
999    scanner.close();
1000    reader.close(cacheConf.shouldEvictOnClose());
1001
1002    // Let's read back the two files to ensure the blocks exactly match
1003    hsf = new HStoreFile(this.fs, pathCowOff, conf, cacheConf, BloomType.NONE, true);
1004    hsf.initReader();
1005    StoreFileReader readerOne = hsf.getReader();
1006    readerOne.loadFileInfo();
1007    StoreFileScanner scannerOne = getStoreFileScanner(readerOne, true, true);
1008    scannerOne.seek(KeyValue.LOWESTKEY);
1009    hsf = new HStoreFile(this.fs, pathCowOn, conf, cacheConf, BloomType.NONE, true);
1010    hsf.initReader();
1011    StoreFileReader readerTwo = hsf.getReader();
1012    readerTwo.loadFileInfo();
1013    StoreFileScanner scannerTwo = getStoreFileScanner(readerTwo, true, true);
1014    scannerTwo.seek(KeyValue.LOWESTKEY);
1015    Cell kv1 = null;
1016    Cell kv2 = null;
1017    while ((kv1 = scannerOne.next()) != null) {
1018      kv2 = scannerTwo.next();
1019      assertTrue(kv1.equals(kv2));
1020      KeyValue keyv1 = KeyValueUtil.ensureKeyValue(kv1);
1021      KeyValue keyv2 = KeyValueUtil.ensureKeyValue(kv2);
1022      assertTrue(Bytes.compareTo(keyv1.getBuffer(), keyv1.getKeyOffset(), keyv1.getKeyLength(),
1023        keyv2.getBuffer(), keyv2.getKeyOffset(), keyv2.getKeyLength()) == 0);
1024      assertTrue(Bytes.compareTo(kv1.getValueArray(), kv1.getValueOffset(), kv1.getValueLength(),
1025        kv2.getValueArray(), kv2.getValueOffset(), kv2.getValueLength()) == 0);
1026    }
1027    assertNull(scannerTwo.next());
1028    assertEquals(startHit + 6, cs.getHitCount());
1029    assertEquals(startMiss, cs.getMissCount());
1030    assertEquals(startEvicted, cs.getEvictedCount());
1031    startHit += 6;
1032    scannerOne.close();
1033    readerOne.close(cacheConf.shouldEvictOnClose());
1034    scannerTwo.close();
1035    readerTwo.close(cacheConf.shouldEvictOnClose());
1036
1037    // Let's close the first file with evict on close turned on
1038    conf.setBoolean("hbase.rs.evictblocksonclose", true);
1039    cacheConf = new CacheConfig(conf, bc);
1040    hsf = new HStoreFile(this.fs, pathCowOff, conf, cacheConf, BloomType.NONE, true);
1041    hsf.initReader();
1042    reader = hsf.getReader();
1043    reader.close(cacheConf.shouldEvictOnClose());
1044
1045    // We should have 3 new evictions but the evict count stat should not change. Eviction because
1046    // of HFile invalidation is not counted along with normal evictions
1047    assertEquals(startHit, cs.getHitCount());
1048    assertEquals(startMiss, cs.getMissCount());
1049    assertEquals(startEvicted, cs.getEvictedCount());
1050
1051    // Let's close the second file with evict on close turned off
1052    conf.setBoolean("hbase.rs.evictblocksonclose", false);
1053    cacheConf = new CacheConfig(conf, bc);
1054    hsf = new HStoreFile(this.fs, pathCowOn, conf, cacheConf, BloomType.NONE, true);
1055    hsf.initReader();
1056    reader = hsf.getReader();
1057    reader.close(cacheConf.shouldEvictOnClose());
1058
1059    // We expect no changes
1060    assertEquals(startHit, cs.getHitCount());
1061    assertEquals(startMiss, cs.getMissCount());
1062    assertEquals(startEvicted, cs.getEvictedCount());
1063  }
1064
1065  private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo hri,
1066    final String family, final HStoreFile sf, final byte[] splitKey, boolean isTopRef)
1067    throws IOException {
1068    Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null);
1069    if (null == path) {
1070      return null;
1071    }
1072    List<Path> splitFiles = new ArrayList<>();
1073    splitFiles.add(path);
1074    MasterProcedureEnv mockEnv = mock(MasterProcedureEnv.class);
1075    MasterServices mockServices = mock(MasterServices.class);
1076    when(mockEnv.getMasterServices()).thenReturn(mockServices);
1077    when(mockEnv.getMasterConfiguration()).thenReturn(new Configuration());
1078    TableDescriptors mockTblDescs = mock(TableDescriptors.class);
1079    when(mockServices.getTableDescriptors()).thenReturn(mockTblDescs);
1080    TableDescriptor mockTblDesc = TableDescriptorBuilder.newBuilder(hri.getTable())
1081      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
1082    when(mockTblDescs.get(any())).thenReturn(mockTblDesc);
1083    Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv);
1084    return new Path(new Path(regionDir, family), path.getName());
1085  }
1086
1087  private StoreFileWriter writeStoreFile(Configuration conf, CacheConfig cacheConf, Path path,
1088    int numBlocks) throws IOException {
1089    // Let's put ~5 small KVs in each block, so let's make 5*numBlocks KVs
1090    int numKVs = 5 * numBlocks;
1091    List<KeyValue> kvs = new ArrayList<>(numKVs);
1092    byte[] b = Bytes.toBytes("x");
1093    int totalSize = 0;
1094    for (int i = numKVs; i > 0; i--) {
1095      KeyValue kv = new KeyValue(b, b, b, i, b);
1096      kvs.add(kv);
1097      // kv has memstoreTS 0, which takes 1 byte to store.
1098      totalSize += kv.getLength() + 1;
1099    }
1100    int blockSize = totalSize / numBlocks;
1101    HFileContext meta = new HFileContextBuilder().withBlockSize(blockSize).withChecksumType(CKTYPE)
1102      .withBytesPerCheckSum(CKBYTES).build();
1103    // Make a store file and write data to it.
1104    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
1105      .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build();
1106    // We'll write N-1 KVs to ensure we don't write an extra block
1107    kvs.remove(kvs.size() - 1);
1108    for (KeyValue kv : kvs) {
1109      writer.append(kv);
1110    }
1111    writer.appendMetadata(0, false);
1112    writer.close();
1113    return writer;
1114  }
1115
1116  /**
1117   * Check if data block encoding information is saved correctly in HFile's file info.
1118   */
1119  @Test
1120  public void testDataBlockEncodingMetaData() throws IOException {
1121    // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.
1122    Path dir = new Path(new Path(testDir, "7e0102"), "familyname");
1123    Path path = new Path(dir, "1234567890");
1124
1125    DataBlockEncoding dataBlockEncoderAlgo = DataBlockEncoding.FAST_DIFF;
1126    cacheConf = new CacheConfig(conf);
1127    HFileContext meta =
1128      new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).withChecksumType(CKTYPE)
1129        .withBytesPerCheckSum(CKBYTES).withDataBlockEncoding(dataBlockEncoderAlgo).build();
1130    // Make a store file and write data to it.
1131    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
1132      .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build();
1133    writer.close();
1134
1135    HStoreFile storeFile =
1136      new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);
1137    storeFile.initReader();
1138    StoreFileReader reader = storeFile.getReader();
1139
1140    Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
1141    byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);
1142    assertArrayEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
1143  }
1144}