001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.DataInput;
027import java.io.DataOutput;
028import java.io.IOException;
029import java.nio.ByteBuffer;
030import java.util.Arrays;
031import java.util.Objects;
032import java.util.Random;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FSDataInputStream;
035import org.apache.hadoop.fs.FSDataOutputStream;
036import org.apache.hadoop.fs.FileStatus;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.ArrayBackedTag;
040import org.apache.hadoop.hbase.ByteBufferKeyValue;
041import org.apache.hadoop.hbase.Cell;
042import org.apache.hadoop.hbase.CellComparatorImpl;
043import org.apache.hadoop.hbase.CellUtil;
044import org.apache.hadoop.hbase.HBaseClassTestRule;
045import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
046import org.apache.hadoop.hbase.HBaseTestingUtility;
047import org.apache.hadoop.hbase.HConstants;
048import org.apache.hadoop.hbase.KeyValue;
049import org.apache.hadoop.hbase.KeyValue.Type;
050import org.apache.hadoop.hbase.KeyValueUtil;
051import org.apache.hadoop.hbase.PrivateCellUtil;
052import org.apache.hadoop.hbase.Tag;
053import org.apache.hadoop.hbase.io.compress.Compression;
054import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
055import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
056import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
057import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
058import org.apache.hadoop.hbase.nio.ByteBuff;
059import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
060import org.apache.hadoop.hbase.testclassification.IOTests;
061import org.apache.hadoop.hbase.testclassification.SmallTests;
062import org.apache.hadoop.hbase.util.ByteBufferUtils;
063import org.apache.hadoop.hbase.util.Bytes;
064import org.apache.hadoop.io.Writable;
065import org.junit.BeforeClass;
066import org.junit.ClassRule;
067import org.junit.Rule;
068import org.junit.Test;
069import org.junit.experimental.categories.Category;
070import org.junit.rules.TestName;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074/**
075 * test hfile features.
076 */
077@Category({IOTests.class, SmallTests.class})
078public class TestHFile  {
079
080  @ClassRule
081  public static final HBaseClassTestRule CLASS_RULE =
082      HBaseClassTestRule.forClass(TestHFile.class);
083
084  @Rule public TestName testName = new TestName();
085
086  private static final Logger LOG = LoggerFactory.getLogger(TestHFile.class);
087  private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2;
088  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
089  private static String ROOT_DIR =
090    TEST_UTIL.getDataTestDir("TestHFile").toString();
091  private final int minBlockSize = 512;
092  private static String localFormatter = "%010d";
093  private static CacheConfig cacheConf = null;
094  private static Configuration conf ;
095  private static FileSystem fs;
096
097  @BeforeClass
098  public static void setUp() throws Exception {
099    conf = TEST_UTIL.getConfiguration();
100    fs = TEST_UTIL.getTestFileSystem();
101  }
102
103  @Test
104  public void testReaderWithoutBlockCache() throws Exception {
105     Path path = writeStoreFile();
106     try{
107       readStoreFile(path);
108     } catch (Exception e) {
109       // fail test
110       assertTrue(false);
111     }
112  }
113
114
115  private void readStoreFile(Path storeFilePath) throws Exception {
116    // Open the file reader with block cache disabled.
117    HFile.Reader reader = HFile.createReader(fs, storeFilePath, conf);
118    long offset = 0;
119    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
120      HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null);
121      offset += block.getOnDiskSizeWithHeader();
122    }
123  }
124
125  private Path writeStoreFile() throws IOException {
126    Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile");
127    HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
128    StoreFileWriter sfw =
129        new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir)
130            .withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build();
131
132    final int rowLen = 32;
133    Random RNG = new Random();
134    for (int i = 0; i < 1000; ++i) {
135      byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i);
136      byte[] v = RandomKeyValueUtil.randomValue(RNG);
137      int cfLen = RNG.nextInt(k.length - rowLen + 1);
138      KeyValue kv =
139          new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen,
140              k.length - rowLen - cfLen, RNG.nextLong(), generateKeyType(RNG), v, 0, v.length);
141      sfw.append(kv);
142    }
143
144    sfw.close();
145    return sfw.getPath();
146  }
147
148  public static KeyValue.Type generateKeyType(Random rand) {
149    if (rand.nextBoolean()) {
150      // Let's make half of KVs puts.
151      return KeyValue.Type.Put;
152    } else {
153      KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
154      if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) {
155        throw new RuntimeException("Generated an invalid key type: " + keyType + ". "
156            + "Probably the layout of KeyValue.Type has changed.");
157      }
158      return keyType;
159    }
160  }
161
162  /**
163   * Test empty HFile.
164   * Test all features work reasonably when hfile is empty of entries.
165   * @throws IOException
166   */
167  @Test
168  public void testEmptyHFile() throws IOException {
169    if (cacheConf == null) cacheConf = new CacheConfig(conf);
170    Path f = new Path(ROOT_DIR, testName.getMethodName());
171    HFileContext context = new HFileContextBuilder().withIncludesTags(false).build();
172    Writer w =
173        HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create();
174    w.close();
175    Reader r = HFile.createReader(fs, f, cacheConf, true, conf);
176    r.loadFileInfo();
177    assertFalse(r.getFirstKey().isPresent());
178    assertFalse(r.getLastKey().isPresent());
179  }
180
181  /**
182   * Create 0-length hfile and show that it fails
183   */
184  @Test
185  public void testCorrupt0LengthHFile() throws IOException {
186    if (cacheConf == null) cacheConf = new CacheConfig(conf);
187    Path f = new Path(ROOT_DIR, testName.getMethodName());
188    FSDataOutputStream fsos = fs.create(f);
189    fsos.close();
190
191    try {
192      Reader r = HFile.createReader(fs, f, cacheConf, true, conf);
193    } catch (CorruptHFileException che) {
194      // Expected failure
195      return;
196    }
197    fail("Should have thrown exception");
198  }
199
200  public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
201    FileStatus fst = fs.getFileStatus(src);
202    long len = fst.getLen();
203    len = len / 2 ;
204
205    // create a truncated hfile
206    FSDataOutputStream fdos = fs.create(dst);
207    byte[] buf = new byte[(int)len];
208    FSDataInputStream fdis = fs.open(src);
209    fdis.read(buf);
210    fdos.write(buf);
211    fdis.close();
212    fdos.close();
213  }
214
215  /**
216   * Create a truncated hfile and verify that exception thrown.
217   */
218  @Test
219  public void testCorruptTruncatedHFile() throws IOException {
220    if (cacheConf == null) cacheConf = new CacheConfig(conf);
221    Path f = new Path(ROOT_DIR, testName.getMethodName());
222    HFileContext  context = new HFileContextBuilder().build();
223    Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f)
224        .withFileContext(context).create();
225    writeSomeRecords(w, 0, 100, false);
226    w.close();
227
228    Path trunc = new Path(f.getParent(), "trucated");
229    truncateFile(fs, w.getPath(), trunc);
230
231    try {
232      Reader r = HFile.createReader(fs, trunc, cacheConf, true, conf);
233    } catch (CorruptHFileException che) {
234      // Expected failure
235      return;
236    }
237    fail("Should have thrown exception");
238  }
239
240  // write some records into the hfile
241  // write them twice
242  private int writeSomeRecords(Writer writer, int start, int n, boolean useTags)
243      throws IOException {
244    String value = "value";
245    KeyValue kv;
246    for (int i = start; i < (start + n); i++) {
247      String key = String.format(localFormatter, Integer.valueOf(i));
248      if (useTags) {
249        Tag t = new ArrayBackedTag((byte) 1, "myTag1");
250        Tag[] tags = new Tag[1];
251        tags[0] = t;
252        kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
253            HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags);
254        writer.append(kv);
255      } else {
256        kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
257            Bytes.toBytes(value + key));
258        writer.append(kv);
259      }
260    }
261    return (start + n);
262  }
263
264  private void readAllRecords(HFileScanner scanner) throws IOException {
265    readAndCheckbytes(scanner, 0, 100);
266  }
267
268  // read the records and check
269  private int readAndCheckbytes(HFileScanner scanner, int start, int n)
270      throws IOException {
271    String value = "value";
272    int i = start;
273    for (; i < (start + n); i++) {
274      ByteBuffer key = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey());
275      ByteBuffer val = scanner.getValue();
276      String keyStr = String.format(localFormatter, Integer.valueOf(i));
277      String valStr = value + keyStr;
278      KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"),
279          Bytes.toBytes("qual"), Bytes.toBytes(valStr));
280      byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0,
281          Bytes.toBytes(key).length).getKey();
282      assertTrue("bytes for keys do not match " + keyStr + " " +
283        Bytes.toString(Bytes.toBytes(key)),
284          Arrays.equals(kv.getKey(), keyBytes));
285      byte [] valBytes = Bytes.toBytes(val);
286      assertTrue("bytes for vals do not match " + valStr + " " +
287        Bytes.toString(valBytes),
288        Arrays.equals(Bytes.toBytes(valStr), valBytes));
289      if (!scanner.next()) {
290        break;
291      }
292    }
293    assertEquals(i, start + n - 1);
294    return (start + n);
295  }
296
297  private byte[] getSomeKey(int rowId) {
298    KeyValue kv = new KeyValue(String.format(localFormatter, Integer.valueOf(rowId)).getBytes(),
299        Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put);
300    return kv.getKey();
301  }
302
303  private void writeRecords(Writer writer, boolean useTags) throws IOException {
304    writeSomeRecords(writer, 0, 100, useTags);
305    writer.close();
306  }
307
308  private FSDataOutputStream createFSOutput(Path name) throws IOException {
309    //if (fs.exists(name)) fs.delete(name, true);
310    FSDataOutputStream fout = fs.create(name);
311    return fout;
312  }
313
314  /**
315   * test none codecs
316   * @param useTags
317   */
318  void basicWithSomeCodec(String codec, boolean useTags) throws IOException {
319    if (useTags) {
320      conf.setInt("hfile.format.version", 3);
321    }
322    if (cacheConf == null) cacheConf = new CacheConfig(conf);
323    Path  ncHFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags);
324    FSDataOutputStream fout = createFSOutput(ncHFile);
325    HFileContext meta = new HFileContextBuilder()
326                        .withBlockSize(minBlockSize)
327                        .withCompression(HFileWriterImpl.compressionByName(codec))
328                        .build();
329    Writer writer = HFile.getWriterFactory(conf, cacheConf)
330        .withOutputStream(fout)
331        .withFileContext(meta)
332        .withComparator(CellComparatorImpl.COMPARATOR)
333        .create();
334    LOG.info(Objects.toString(writer));
335    writeRecords(writer, useTags);
336    fout.close();
337    FSDataInputStream fin = fs.open(ncHFile);
338    Reader reader = HFile.createReaderFromStream(ncHFile, fs.open(ncHFile),
339      fs.getFileStatus(ncHFile).getLen(), cacheConf, conf);
340    System.out.println(cacheConf.toString());
341    // Load up the index.
342    reader.loadFileInfo();
343    // Get a scanner that caches and that does not use pread.
344    HFileScanner scanner = reader.getScanner(true, false);
345    // Align scanner at start of the file.
346    scanner.seekTo();
347    readAllRecords(scanner);
348    int seekTo = scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50)));
349    System.out.println(seekTo);
350    assertTrue("location lookup failed",
351        scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))) == 0);
352    // read the key and see if it matches
353    ByteBuffer readKey = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey());
354    assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
355      Bytes.toBytes(readKey)));
356
357    scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0)));
358    ByteBuffer val1 = scanner.getValue();
359    scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0)));
360    ByteBuffer val2 = scanner.getValue();
361    assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
362
363    reader.close();
364    fin.close();
365    fs.delete(ncHFile, true);
366  }
367
368  @Test
369  public void testTFileFeatures() throws IOException {
370    testHFilefeaturesInternals(false);
371    testHFilefeaturesInternals(true);
372  }
373
374  protected void testHFilefeaturesInternals(boolean useTags) throws IOException {
375    basicWithSomeCodec("none", useTags);
376    basicWithSomeCodec("gz", useTags);
377  }
378
379  private void writeNumMetablocks(Writer writer, int n) {
380    for (int i = 0; i < n; i++) {
381      writer.appendMetaBlock("HFileMeta" + i, new Writable() {
382        private int val;
383        public Writable setVal(int val) { this.val = val; return this; }
384
385        @Override
386        public void write(DataOutput out) throws IOException {
387          out.write(("something to test" + val).getBytes());
388        }
389
390        @Override
391        public void readFields(DataInput in) throws IOException { }
392      }.setVal(i));
393    }
394  }
395
396  private void someTestingWithMetaBlock(Writer writer) {
397    writeNumMetablocks(writer, 10);
398  }
399
400  private void readNumMetablocks(Reader reader, int n) throws IOException {
401    for (int i = 0; i < n; i++) {
402      ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader();
403      ByteBuffer expected =
404        ByteBuffer.wrap(("something to test" + i).getBytes());
405      assertEquals(
406          "failed to match metadata",
407          Bytes.toStringBinary(expected),
408          Bytes.toStringBinary(actual.array(), actual.arrayOffset() + actual.position(),
409              actual.capacity()));
410    }
411  }
412
413  private void someReadingWithMetaBlock(Reader reader) throws IOException {
414    readNumMetablocks(reader, 10);
415  }
416
417  private void metablocks(final String compress) throws Exception {
418    if (cacheConf == null) cacheConf = new CacheConfig(conf);
419    Path mFile = new Path(ROOT_DIR, "meta.hfile");
420    FSDataOutputStream fout = createFSOutput(mFile);
421    HFileContext meta = new HFileContextBuilder()
422                        .withCompression(HFileWriterImpl.compressionByName(compress))
423                        .withBlockSize(minBlockSize).build();
424    Writer writer = HFile.getWriterFactory(conf, cacheConf)
425        .withOutputStream(fout)
426        .withFileContext(meta)
427        .create();
428    someTestingWithMetaBlock(writer);
429    writer.close();
430    fout.close();
431    FSDataInputStream fin = fs.open(mFile);
432    Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
433        this.fs.getFileStatus(mFile).getLen(), cacheConf, conf);
434    reader.loadFileInfo();
435    // No data -- this should return false.
436    assertFalse(reader.getScanner(false, false).seekTo());
437    someReadingWithMetaBlock(reader);
438    fs.delete(mFile, true);
439    reader.close();
440    fin.close();
441  }
442
443  // test meta blocks for hfiles
444  @Test
445  public void testMetaBlocks() throws Exception {
446    metablocks("none");
447    metablocks("gz");
448  }
449
450  @Test
451  public void testNullMetaBlocks() throws Exception {
452    if (cacheConf == null) cacheConf = new CacheConfig(conf);
453    for (Compression.Algorithm compressAlgo :
454        HBaseCommonTestingUtility.COMPRESSION_ALGORITHMS) {
455      Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
456      FSDataOutputStream fout = createFSOutput(mFile);
457      HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo)
458                          .withBlockSize(minBlockSize).build();
459      Writer writer = HFile.getWriterFactory(conf, cacheConf)
460          .withOutputStream(fout)
461          .withFileContext(meta)
462          .create();
463      KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes());
464      writer.append(kv);
465      writer.close();
466      fout.close();
467      Reader reader = HFile.createReader(fs, mFile, cacheConf, true, conf);
468      reader.loadFileInfo();
469      assertNull(reader.getMetaBlock("non-existant", false));
470    }
471  }
472
473  /**
474   * Make sure the ordinals for our compression algorithms do not change on us.
475   */
476  @Test
477  public void testCompressionOrdinance() {
478    assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
479    assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
480    assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
481    assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
482    assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
483  }
484
485  @Test
486  public void testShortMidpointSameQual() {
487    Cell left = CellUtil.createCell(Bytes.toBytes("a"),
488        Bytes.toBytes("a"),
489        Bytes.toBytes("a"),
490        11,
491        KeyValue.Type.Maximum.getCode(),
492        HConstants.EMPTY_BYTE_ARRAY);
493    Cell right = CellUtil.createCell(Bytes.toBytes("a"),
494        Bytes.toBytes("a"),
495        Bytes.toBytes("a"),
496        9,
497        KeyValue.Type.Maximum.getCode(),
498        HConstants.EMPTY_BYTE_ARRAY);
499    Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
500    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0);
501    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0);
502  }
503
504  @Test
505  public void testGetShortMidpoint() {
506    Cell left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
507    Cell right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
508    Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
509    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0);
510    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0);
511
512    left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
513    right = CellUtil.createCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a"));
514    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
515    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
516    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0);
517
518    left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a"));
519    right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a"));
520    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
521    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
522    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0);
523
524    left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
525    right = CellUtil.createCell(Bytes.toBytes("bbbbbbb"), Bytes.toBytes("a"), Bytes.toBytes("a"));
526    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
527    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
528    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0);
529    assertEquals(1, mid.getRowLength());
530
531    left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
532    right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("a"));
533    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
534    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
535    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0);
536
537    left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
538    right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaa"), Bytes.toBytes("b"));
539    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
540    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
541    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0);
542    assertEquals(2, mid.getFamilyLength());
543
544    left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
545    right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa"));
546    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
547    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
548    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) < 0);
549    assertEquals(2, mid.getQualifierLength());
550
551    left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a"));
552    right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b"));
553    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
554    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
555    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) <= 0);
556    assertEquals(1, mid.getQualifierLength());
557
558    // Assert that if meta comparator, it returns the right cell -- i.e. no
559    // optimization done.
560    left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a"));
561    right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a"));
562    mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.META_COMPARATOR, left, right);
563    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) < 0);
564    assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0);
565
566    /**
567     * See HBASE-7845
568     */
569    byte[] rowA = Bytes.toBytes("rowA");
570    byte[] rowB = Bytes.toBytes("rowB");
571
572    byte[] family = Bytes.toBytes("family");
573    byte[] qualA = Bytes.toBytes("qfA");
574    byte[] qualB = Bytes.toBytes("qfB");
575    final CellComparatorImpl keyComparator = CellComparatorImpl.COMPARATOR;
576    // verify that faked shorter rowkey could be generated
577    long ts = 5;
578    KeyValue kv1 = new KeyValue(Bytes.toBytes("the quick brown fox"), family, qualA, ts, Type.Put);
579    KeyValue kv2 = new KeyValue(Bytes.toBytes("the who test text"), family, qualA, ts, Type.Put);
580    Cell newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
581    assertTrue(keyComparator.compare(kv1, newKey) < 0);
582    assertTrue((keyComparator.compare(kv2, newKey)) > 0);
583    byte[] expectedArray = Bytes.toBytes("the r");
584    Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray,
585        0, expectedArray.length);
586
587    // verify: same with "row + family + qualifier", return rightKey directly
588    kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put);
589    kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 0, Type.Put);
590    assertTrue(keyComparator.compare(kv1, kv2) < 0);
591    newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
592    assertTrue(keyComparator.compare(kv1, newKey) < 0);
593    assertTrue((keyComparator.compare(kv2, newKey)) == 0);
594    kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -5, Type.Put);
595    kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -10, Type.Put);
596    assertTrue(keyComparator.compare(kv1, kv2) < 0);
597    newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
598    assertTrue(keyComparator.compare(kv1, newKey) < 0);
599    assertTrue((keyComparator.compare(kv2, newKey)) == 0);
600
601    // verify: same with row, different with qualifier
602    kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put);
603    kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualB, 5, Type.Put);
604    assertTrue(keyComparator.compare(kv1, kv2) < 0);
605    newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
606    assertTrue(keyComparator.compare(kv1, newKey) < 0);
607    assertTrue((keyComparator.compare(kv2, newKey)) > 0);
608    assertTrue(Arrays.equals(CellUtil.cloneFamily(newKey), family));
609    assertTrue(Arrays.equals(CellUtil.cloneQualifier(newKey), qualB));
610    assertTrue(newKey.getTimestamp() == HConstants.LATEST_TIMESTAMP);
611    assertTrue(newKey.getTypeByte() == Type.Maximum.getCode());
612
613    // verify metaKeyComparator's getShortMidpointKey output
614    final CellComparatorImpl metaKeyComparator = CellComparatorImpl.META_COMPARATOR;
615    kv1 = new KeyValue(Bytes.toBytes("ilovehbase123"), family, qualA, 5, Type.Put);
616    kv2 = new KeyValue(Bytes.toBytes("ilovehbase234"), family, qualA, 0, Type.Put);
617    newKey = HFileWriterImpl.getMidpoint(metaKeyComparator, kv1, kv2);
618    assertTrue(metaKeyComparator.compare(kv1, newKey) < 0);
619    assertTrue((metaKeyComparator.compare(kv2, newKey) == 0));
620
621    // verify common fix scenario
622    kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, ts, Type.Put);
623    kv2 = new KeyValue(Bytes.toBytes("ilovehbaseandhdfs"), family, qualA, ts, Type.Put);
624    assertTrue(keyComparator.compare(kv1, kv2) < 0);
625    newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
626    assertTrue(keyComparator.compare(kv1, newKey) < 0);
627    assertTrue((keyComparator.compare(kv2, newKey)) > 0);
628    expectedArray = Bytes.toBytes("ilovehbasea");
629    Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray,
630        0, expectedArray.length);
631    // verify only 1 offset scenario
632    kv1 = new KeyValue(Bytes.toBytes("100abcdefg"), family, qualA, ts, Type.Put);
633    kv2 = new KeyValue(Bytes.toBytes("101abcdefg"), family, qualA, ts, Type.Put);
634    assertTrue(keyComparator.compare(kv1, kv2) < 0);
635    newKey = HFileWriterImpl.getMidpoint(keyComparator, kv1, kv2);
636    assertTrue(keyComparator.compare(kv1, newKey) < 0);
637    assertTrue((keyComparator.compare(kv2, newKey)) > 0);
638    expectedArray = Bytes.toBytes("101");
639    Bytes.equals(newKey.getRowArray(), newKey.getRowOffset(), newKey.getRowLength(), expectedArray,
640        0, expectedArray.length);
641  }
642
643  @Test
644  public void testDBEShipped() throws IOException {
645    for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
646      DataBlockEncoder encoder = encoding.getEncoder();
647      if (encoder == null) {
648        continue;
649      }
650      Path f = new Path(ROOT_DIR, testName.getMethodName() + "_" + encoding);
651      HFileContext context = new HFileContextBuilder()
652          .withIncludesTags(false)
653          .withDataBlockEncoding(encoding).build();
654      HFileWriterImpl writer = (HFileWriterImpl) HFile.getWriterFactory(conf, cacheConf)
655          .withPath(fs, f).withFileContext(context).create();
656
657      KeyValue kv = new KeyValue(Bytes.toBytes("testkey1"), Bytes.toBytes("family"),
658          Bytes.toBytes("qual"), Bytes.toBytes("testvalue"));
659      KeyValue kv2 = new KeyValue(Bytes.toBytes("testkey2"), Bytes.toBytes("family"),
660        Bytes.toBytes("qual"), Bytes.toBytes("testvalue"));
661      KeyValue kv3 = new KeyValue(Bytes.toBytes("testkey3"), Bytes.toBytes("family"),
662        Bytes.toBytes("qual"), Bytes.toBytes("testvalue"));
663
664      ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer());
665      ByteBuffer buffer2 = ByteBuffer.wrap(kv2.getBuffer());
666      ByteBuffer buffer3 = ByteBuffer.wrap(kv3.getBuffer());
667
668      writer.append(new ByteBufferKeyValue(buffer, 0, buffer.remaining()));
669      writer.beforeShipped();
670
671      // pollute first cell's backing ByteBuffer
672      ByteBufferUtils.copyFromBufferToBuffer(buffer3, buffer);
673
674      // write another cell, if DBE not Shipped, test will fail
675      writer.append(new ByteBufferKeyValue(buffer2, 0, buffer2.remaining()));
676      writer.close();
677    }
678  }
679}