001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.io.IOException;
021import org.apache.hadoop.conf.Configuration;
022import org.apache.hadoop.fs.FileSystem;
023import org.apache.hadoop.fs.Path;
024import org.apache.hadoop.hbase.ArrayBackedTag;
025import org.apache.hadoop.hbase.HBaseConfiguration;
026import org.apache.hadoop.hbase.HBaseTestingUtil;
027import org.apache.hadoop.hbase.KeyValue;
028import org.apache.hadoop.hbase.Tag;
029import org.apache.hadoop.hbase.io.compress.Compression;
030import org.apache.hadoop.hbase.io.hfile.HFileContext;
031import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
032import org.apache.hadoop.hbase.testclassification.MiscTests;
033import org.apache.hadoop.hbase.testclassification.SmallTests;
034import org.apache.hadoop.hbase.util.Bytes;
035import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
036import org.junit.jupiter.api.BeforeEach;
037import org.junit.jupiter.api.Test;
038
039/**
040 * Test DataBlockEncodingTool.
041 */
042@org.junit.jupiter.api.Tag(MiscTests.TAG)
043@org.junit.jupiter.api.Tag(SmallTests.TAG)
044public class TestDataBlockEncodingTool {
045
046  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
047  private static final String ROOT_DIR =
048    TEST_UTIL.getDataTestDir("TestDataBlockEncodingTool").toString();
049  private static final Configuration conf = TEST_UTIL.getConfiguration();
050  private static FileSystem fs;
051  private static StoreFileWriter sfw;
052
053  @BeforeEach
054  public void setUp() throws IOException {
055    fs = TEST_UTIL.getTestFileSystem();
056  }
057
058  private void testHFile(String fileName, boolean useTags, boolean allTags) throws IOException {
059    Path path = new Path(ROOT_DIR, fileName);
060    try {
061      createHFileWithTags(path, useTags, allTags);
062      testDataBlockingTool(path);
063    } finally {
064      if (fs.exists(path)) {
065        fs.delete(path, false);
066      }
067    }
068  }
069
070  private void createHFileWithTags(Path path, boolean useTags, boolean allTags) throws IOException {
071    HFileContext meta =
072      new HFileContextBuilder().withBlockSize(64 * 1024).withIncludesTags(useTags).build();
073    sfw = new StoreFileWriter.Builder(conf, fs).withFilePath(path).withFileContext(meta).build();
074    long now = EnvironmentEdgeManager.currentTime();
075    byte[] FAMILY = Bytes.toBytes("cf");
076    byte[] QUALIFIER = Bytes.toBytes("q");
077    try {
078      for (char d = 'a'; d <= 'z'; d++) {
079        for (char e = 'a'; e <= 'z'; e++) {
080          byte[] b = new byte[] { (byte) d, (byte) e };
081          KeyValue kv;
082          if (useTags) {
083            if (allTags) {
084              // Write cells with tags to HFile.
085              Tag[] tags = new Tag[] { new ArrayBackedTag((byte) 0, Bytes.toString(b)),
086                new ArrayBackedTag((byte) 0, Bytes.toString(b)) };
087              kv = new KeyValue(b, FAMILY, QUALIFIER, now, b, tags);
088            } else {
089              // Write half cells with tags and half without tags to HFile.
090              if ((e - 'a') % 2 == 0) {
091                kv = new KeyValue(b, FAMILY, QUALIFIER, now, b);
092              } else {
093                Tag[] tags = new Tag[] { new ArrayBackedTag((byte) 0, Bytes.toString(b)),
094                  new ArrayBackedTag((byte) 0, Bytes.toString(b)) };
095                kv = new KeyValue(b, FAMILY, QUALIFIER, now, b, tags);
096              }
097            }
098          } else {
099            // Write cells without tags to HFile.
100            kv = new KeyValue(b, FAMILY, QUALIFIER, now, b);
101          }
102          sfw.append(kv);
103        }
104      }
105      sfw.appendMetadata(0, false);
106    } finally {
107      sfw.close();
108    }
109  }
110
111  private static void testDataBlockingTool(Path path) throws IOException {
112    Configuration conf = HBaseConfiguration.create();
113    int maxKV = Integer.MAX_VALUE;
114    boolean doVerify = true;
115    boolean doBenchmark = true;
116    String testHFilePath = path.toString();
117    DataBlockEncodingTool.testCodecs(conf, maxKV, testHFilePath, Compression.Algorithm.GZ.getName(),
118      doBenchmark, doVerify);
119  }
120
121  @Test
122  public void testHFileAllCellsWithTags() throws IOException {
123    testHFile("1234567890", true, true);
124  }
125
126  @Test
127  public void testHFileAllCellsWithoutTags() throws IOException {
128    testHFile("1234567089", false, false);
129  }
130
131  @Test
132  public void testHFileHalfCellsWithTags() throws IOException {
133    testHFile("1234560789", true, false);
134  }
135}