001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.*;
021
022import java.io.BufferedOutputStream;
023import java.io.DataOutputStream;
024import java.io.IOException;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.hbase.HBaseClassTestRule;
027import org.apache.hadoop.hbase.io.compress.Compression;
028import org.apache.hadoop.hbase.testclassification.MiscTests;
029import org.apache.hadoop.hbase.testclassification.SmallTests;
030import org.apache.hadoop.io.DataOutputBuffer;
031import org.apache.hadoop.io.compress.CompressionCodec;
032import org.apache.hadoop.io.compress.CompressionOutputStream;
033import org.apache.hadoop.util.NativeCodeLoader;
034import org.apache.hadoop.util.ReflectionUtils;
035import org.junit.ClassRule;
036import org.junit.Test;
037import org.junit.experimental.categories.Category;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041@Category({ MiscTests.class, SmallTests.class })
042public class TestCompressionTest {
043
044  @ClassRule
045  public static final HBaseClassTestRule CLASS_RULE =
046    HBaseClassTestRule.forClass(TestCompressionTest.class);
047
048  private static final Logger LOG = LoggerFactory.getLogger(TestCompressionTest.class);
049
050  @Test
051  public void testExceptionCaching() {
052    // This test will fail if you run the tests with LZO compression available.
053    try {
054      CompressionTest.testCompression(Compression.Algorithm.LZO);
055      fail(); // always throws
056    } catch (IOException e) {
057      // there should be a 'cause'.
058      assertNotNull(e.getCause());
059    }
060
061    // this is testing the caching of the test results.
062    try {
063      CompressionTest.testCompression(Compression.Algorithm.LZO);
064      fail(); // always throws
065    } catch (IOException e) {
066      // there should be NO cause because it's a direct exception not wrapped
067      assertNull(e.getCause());
068    }
069
070    assertFalse(CompressionTest.testCompression("LZO"));
071  }
072
073  @Test
074  public void testTestCompression() {
075    assertTrue(CompressionTest.testCompression("NONE"));
076    assertTrue(CompressionTest.testCompression("GZ"));
077
078    if (NativeCodeLoader.isNativeCodeLoaded()) {
079      // LZO is GPL so not included in hadoop install. You need to do an extra install to pick
080      // up the needed support. This article is good on the steps needed to add LZO support:
081      // https://stackoverflow.com/questions/23441142/class-com-hadoop-compression-lzo-lzocodec-not-found-for-spark-on-cdh-5
082      // Its unlikely at test time that the extras are installed so this test is useless.
083      // nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
084      nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
085      nativeCodecTest("SNAPPY", "snappy", "org.apache.hadoop.io.compress.SnappyCodec");
086      nativeCodecTest("BZIP2", "bzip2", "org.apache.hadoop.io.compress.BZip2Codec");
087      nativeCodecTest("ZSTD", "zstd", "org.apache.hadoop.io.compress.ZStandardCodec");
088    } else {
089      // Hadoop nativelib is not available
090      LOG.debug("Native code not loaded");
091      // This check is useless as it fails with
092      // ...DoNotRetryIOException: Compression algorithm 'lzo' previously failed test.
093      // assertFalse("LZO", CompressionTest.testCompression("LZO"));
094      // LZ4 requires that the native lib be present before 3.3.1. After 3.3.1, hadoop uses
095      // lz4-java which will do java version of lz4 as last resort -- so the below fails before
096      // 3.3.1 but passes at 3.3.1+... so commenting it out. See HADOOP-17292.
097      // assertFalse("LZ4", CompressionTest.testCompression("LZ4"));
098      // Same thing happens for snappy. See HADOOP-17125
099      // assertFalse(CompressionTest.testCompression("SNAPPY"));
100      assertFalse(CompressionTest.testCompression("BZIP2"));
101      assertFalse(CompressionTest.testCompression("ZSTD"));
102    }
103  }
104
105  private boolean isCompressionAvailable(String codecClassName) {
106    try {
107      Thread.currentThread().getContextClassLoader().loadClass(codecClassName);
108      return true;
109    } catch (Exception ex) {
110      return false;
111    }
112  }
113
114  /**
115   * Verify CompressionTest.testCompression() on a native codec.
116   */
117  private void nativeCodecTest(String codecName, String libName, String codecClassName) {
118    if (isCompressionAvailable(codecClassName)) {
119      try {
120        if (libName != null) {
121          System.loadLibrary(libName);
122        }
123
124        try {
125          Configuration conf = new Configuration();
126          CompressionCodec codec = (CompressionCodec) ReflectionUtils
127            .newInstance(conf.getClassByName(codecClassName), conf);
128
129          DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
130          CompressionOutputStream deflateFilter = codec.createOutputStream(compressedDataBuffer);
131
132          byte[] data = new byte[1024];
133          DataOutputStream deflateOut =
134            new DataOutputStream(new BufferedOutputStream(deflateFilter));
135          deflateOut.write(data, 0, data.length);
136          deflateOut.flush();
137          deflateFilter.finish();
138
139          // Codec class, codec nativelib and Hadoop nativelib with codec JNIs are present
140          assertTrue(CompressionTest.testCompression(codecName));
141        } catch (UnsatisfiedLinkError e) {
142          // Hadoop nativelib does not have codec JNIs.
143          // cannot assert the codec here because the current logic of
144          // CompressionTest checks only classloading, not the codec
145          // usage.
146          LOG.debug("No JNI for codec '" + codecName + "' " + e.getMessage());
147        } catch (Exception e) {
148          LOG.error(codecName, e);
149        }
150      } catch (UnsatisfiedLinkError e) {
151        // nativelib is not available
152        LOG.debug("Native lib not available: " + codecName);
153        assertFalse(CompressionTest.testCompression(codecName));
154      }
155    } else {
156      // Compression Codec class is not available
157      LOG.debug("Codec class not available: " + codecName);
158      assertFalse(CompressionTest.testCompression(codecName));
159    }
160  }
161}