001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.*;
021
022import java.io.BufferedOutputStream;
023import java.io.DataOutputStream;
024import java.io.IOException;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.hbase.HBaseClassTestRule;
027import org.apache.hadoop.hbase.io.compress.Compression;
028import org.apache.hadoop.hbase.testclassification.MiscTests;
029import org.apache.hadoop.hbase.testclassification.SmallTests;
030import org.apache.hadoop.io.DataOutputBuffer;
031import org.apache.hadoop.io.compress.CompressionCodec;
032import org.apache.hadoop.io.compress.CompressionOutputStream;
033import org.apache.hadoop.util.NativeCodeLoader;
034import org.apache.hadoop.util.ReflectionUtils;
035import org.junit.ClassRule;
036import org.junit.Test;
037import org.junit.experimental.categories.Category;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041@Category({MiscTests.class, SmallTests.class})
042public class TestCompressionTest {
043
044  @ClassRule
045  public static final HBaseClassTestRule CLASS_RULE =
046      HBaseClassTestRule.forClass(TestCompressionTest.class);
047
048  private static final Logger LOG = LoggerFactory.getLogger(TestCompressionTest.class);
049
050  @Test
051  public void testExceptionCaching() {
052    // This test will fail if you run the tests with LZO compression available.
053    try {
054      CompressionTest.testCompression(Compression.Algorithm.LZO);
055      fail(); // always throws
056    } catch (IOException e) {
057      // there should be a 'cause'.
058      assertNotNull(e.getCause());
059    }
060
061    // this is testing the caching of the test results.
062    try {
063      CompressionTest.testCompression(Compression.Algorithm.LZO);
064      fail(); // always throws
065    } catch (IOException e) {
066      // there should be NO cause because it's a direct exception not wrapped
067      assertNull(e.getCause());
068    }
069
070    assertFalse(CompressionTest.testCompression("LZO"));
071  }
072
073  @Test
074  public void testTestCompression() {
075    assertTrue(CompressionTest.testCompression("NONE"));
076    assertTrue(CompressionTest.testCompression("GZ"));
077
078    if (NativeCodeLoader.isNativeCodeLoaded()) {
079      nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
080      nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
081      nativeCodecTest("SNAPPY", "snappy", "org.apache.hadoop.io.compress.SnappyCodec");
082      nativeCodecTest("BZIP2", "bzip2", "org.apache.hadoop.io.compress.BZip2Codec");
083      nativeCodecTest("ZSTD", "zstd", "org.apache.hadoop.io.compress.ZStandardCodec");
084    } else {
085      // Hadoop nativelib is not available
086      LOG.debug("Native code not loaded");
087      assertFalse(CompressionTest.testCompression("LZO"));
088      assertFalse(CompressionTest.testCompression("LZ4"));
089      assertFalse(CompressionTest.testCompression("SNAPPY"));
090      assertFalse(CompressionTest.testCompression("BZIP2"));
091      assertFalse(CompressionTest.testCompression("ZSTD"));
092    }
093  }
094
095  private boolean isCompressionAvailable(String codecClassName) {
096    try {
097      Thread.currentThread().getContextClassLoader().loadClass(codecClassName);
098      return true;
099    } catch (Exception ex) {
100      return false;
101    }
102  }
103
104  /**
105   * Verify CompressionTest.testCompression() on a native codec.
106   */
107  private void nativeCodecTest(String codecName, String libName, String codecClassName) {
108    if (isCompressionAvailable(codecClassName)) {
109      try {
110        if (libName != null) {
111          System.loadLibrary(libName);
112        }
113
114        try {
115            Configuration conf = new Configuration();
116            CompressionCodec codec = (CompressionCodec)
117              ReflectionUtils.newInstance(conf.getClassByName(codecClassName), conf);
118
119            DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
120            CompressionOutputStream deflateFilter = codec.createOutputStream(compressedDataBuffer);
121
122            byte[] data = new byte[1024];
123            DataOutputStream deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
124            deflateOut.write(data, 0, data.length);
125            deflateOut.flush();
126            deflateFilter.finish();
127
128            // Codec class, codec nativelib and Hadoop nativelib with codec JNIs are present
129            assertTrue(CompressionTest.testCompression(codecName));
130        } catch (UnsatisfiedLinkError e) {
131          // Hadoop nativelib does not have codec JNIs.
132          // cannot assert the codec here because the current logic of
133          // CompressionTest checks only classloading, not the codec
134          // usage.
135          LOG.debug("No JNI for codec '" + codecName + "' " + e.getMessage());
136        } catch (Exception e) {
137          LOG.error(codecName, e);
138        }
139      } catch (UnsatisfiedLinkError e) {
140        // nativelib is not available
141        LOG.debug("Native lib not available: " + codecName);
142        assertFalse(CompressionTest.testCompression(codecName));
143      }
144    } else {
145      // Compression Codec class is not available
146      LOG.debug("Codec class not available: " + codecName);
147      assertFalse(CompressionTest.testCompression(codecName));
148    }
149  }
150}
151