001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.jupiter.api.Assertions.assertFalse;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022import static org.junit.jupiter.api.Assertions.assertNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024import static org.junit.jupiter.api.Assertions.fail;
025
026import java.io.BufferedOutputStream;
027import java.io.DataOutputStream;
028import java.io.IOException;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.io.compress.Compression;
031import org.apache.hadoop.hbase.testclassification.MiscTests;
032import org.apache.hadoop.hbase.testclassification.SmallTests;
033import org.apache.hadoop.io.DataOutputBuffer;
034import org.apache.hadoop.io.compress.CompressionCodec;
035import org.apache.hadoop.io.compress.CompressionOutputStream;
036import org.apache.hadoop.util.NativeCodeLoader;
037import org.apache.hadoop.util.ReflectionUtils;
038import org.junit.jupiter.api.Tag;
039import org.junit.jupiter.api.Test;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043@Tag(MiscTests.TAG)
044@Tag(SmallTests.TAG)
045public class TestCompressionTest {
046
047  private static final Logger LOG = LoggerFactory.getLogger(TestCompressionTest.class);
048
049  @Test
050  public void testExceptionCaching() {
051    // This test will fail if you run the tests with LZO compression available.
052    try {
053      CompressionTest.testCompression(Compression.Algorithm.LZO);
054      fail(); // always throws
055    } catch (IOException e) {
056      // there should be a 'cause'.
057      assertNotNull(e.getCause());
058    }
059
060    // this is testing the caching of the test results.
061    try {
062      CompressionTest.testCompression(Compression.Algorithm.LZO);
063      fail(); // always throws
064    } catch (IOException e) {
065      // there should be NO cause because it's a direct exception not wrapped
066      assertNull(e.getCause());
067    }
068
069    assertFalse(CompressionTest.testCompression("LZO"));
070  }
071
072  @Test
073  public void testTestCompression() {
074    assertTrue(CompressionTest.testCompression("NONE"));
075    assertTrue(CompressionTest.testCompression("GZ"));
076
077    if (NativeCodeLoader.isNativeCodeLoaded()) {
078      // LZO is GPL so not included in hadoop install. You need to do an extra install to pick
079      // up the needed support. This article is good on the steps needed to add LZO support:
080      // https://stackoverflow.com/questions/23441142/class-com-hadoop-compression-lzo-lzocodec-not-found-for-spark-on-cdh-5
081      // Its unlikely at test time that the extras are installed so this test is useless.
082      // nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
083      nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
084      nativeCodecTest("SNAPPY", "snappy", "org.apache.hadoop.io.compress.SnappyCodec");
085      nativeCodecTest("BZIP2", "bzip2", "org.apache.hadoop.io.compress.BZip2Codec");
086      nativeCodecTest("ZSTD", "zstd", "org.apache.hadoop.io.compress.ZStandardCodec");
087    } else {
088      // Hadoop nativelib is not available
089      LOG.debug("Native code not loaded");
090      // This check is useless as it fails with
091      // ...DoNotRetryIOException: Compression algorithm 'lzo' previously failed test.
092      // assertFalse("LZO", CompressionTest.testCompression("LZO"));
093      // LZ4 requires that the native lib be present before 3.3.1. After 3.3.1, hadoop uses
094      // lz4-java which will do java version of lz4 as last resort -- so the below fails before
095      // 3.3.1 but passes at 3.3.1+... so commenting it out. See HADOOP-17292.
096      // assertFalse("LZ4", CompressionTest.testCompression("LZ4"));
097      // Same thing happens for snappy. See HADOOP-17125
098      // assertFalse(CompressionTest.testCompression("SNAPPY"));
099      assertFalse(CompressionTest.testCompression("BZIP2"));
100      assertFalse(CompressionTest.testCompression("ZSTD"));
101    }
102  }
103
104  private boolean isCompressionAvailable(String codecClassName) {
105    try {
106      Thread.currentThread().getContextClassLoader().loadClass(codecClassName);
107      return true;
108    } catch (Exception ex) {
109      return false;
110    }
111  }
112
113  /**
114   * Verify CompressionTest.testCompression() on a native codec.
115   */
116  private void nativeCodecTest(String codecName, String libName, String codecClassName) {
117    if (isCompressionAvailable(codecClassName)) {
118      try {
119        if (libName != null) {
120          System.loadLibrary(libName);
121        }
122
123        try {
124          Configuration conf = new Configuration();
125          CompressionCodec codec = (CompressionCodec) ReflectionUtils
126            .newInstance(conf.getClassByName(codecClassName), conf);
127
128          DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
129          CompressionOutputStream deflateFilter = codec.createOutputStream(compressedDataBuffer);
130
131          byte[] data = new byte[1024];
132          DataOutputStream deflateOut =
133            new DataOutputStream(new BufferedOutputStream(deflateFilter));
134          deflateOut.write(data, 0, data.length);
135          deflateOut.flush();
136          deflateFilter.finish();
137
138          // Codec class, codec nativelib and Hadoop nativelib with codec JNIs are present
139          assertTrue(CompressionTest.testCompression(codecName));
140        } catch (UnsatisfiedLinkError e) {
141          // Hadoop nativelib does not have codec JNIs.
142          // cannot assert the codec here because the current logic of
143          // CompressionTest checks only classloading, not the codec
144          // usage.
145          LOG.debug("No JNI for codec '" + codecName + "' " + e.getMessage());
146        } catch (Exception e) {
147          LOG.error(codecName, e);
148        }
149      } catch (UnsatisfiedLinkError e) {
150        // nativelib is not available
151        LOG.debug("Native lib not available: " + codecName);
152        assertFalse(CompressionTest.testCompression(codecName));
153      }
154    } else {
155      // Compression Codec class is not available
156      LOG.debug("Codec class not available: " + codecName);
157      assertFalse(CompressionTest.testCompression(codecName));
158    }
159  }
160}