View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.IOException;
22  import java.util.Locale;
23
24  import org.apache.commons.lang.StringUtils;
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.classification.InterfaceStability;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.Cell;
33  import org.apache.hadoop.hbase.CellComparator;
34  import org.apache.hadoop.hbase.CellUtil;
35  import org.apache.hadoop.hbase.DoNotRetryIOException;
36  import org.apache.hadoop.hbase.HBaseConfiguration;
37  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
38  import org.apache.hadoop.hbase.io.compress.Compression;
39  import org.apache.hadoop.hbase.io.hfile.HFileWriterImpl;
40  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
41  import org.apache.hadoop.hbase.io.hfile.HFile;
42  import org.apache.hadoop.hbase.io.hfile.HFileContext;
43  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
44  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
45  import org.apache.hadoop.io.compress.Compressor;
46
47  /**
48   * Compression validation test.  Checks compression is working.  Be sure to run
49   * on every node in your cluster.
50   */
51  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
52  @InterfaceStability.Evolving
53  public class CompressionTest {
54    private static final Log LOG = LogFactory.getLog(CompressionTest.class);
55
56    public static boolean testCompression(String codec) {
57      codec = codec.toLowerCase(Locale.ROOT);
58
59      Compression.Algorithm a;
60
61      try {
62        a = Compression.getCompressionAlgorithmByName(codec);
63      } catch (IllegalArgumentException e) {
64        LOG.warn("Codec type: " + codec + " is not known");
65        return false;
66      }
67
68      try {
69        testCompression(a);
70        return true;
71      } catch (IOException ignored) {
72        LOG.warn("Can't instantiate codec: " + codec, ignored);
73        return false;
74      }
75    }
76
77    private final static Boolean[] compressionTestResults
78        = new Boolean[Compression.Algorithm.values().length];
79    static {
80      for (int i = 0 ; i < compressionTestResults.length ; ++i) {
81        compressionTestResults[i] = null;
82      }
83    }
84
85    public static void testCompression(Compression.Algorithm algo)
86        throws IOException {
87      if (compressionTestResults[algo.ordinal()] != null) {
88        if (compressionTestResults[algo.ordinal()]) {
89          return ; // already passed test, dont do it again.
90        } else {
91          // failed.
92          throw new DoNotRetryIOException("Compression algorithm '" + algo.getName() + "'" +
93          " previously failed test.");
94        }
95      }
96
97      try {
98        Compressor c = algo.getCompressor();
99        algo.returnCompressor(c);
100       compressionTestResults[algo.ordinal()] = true; // passes
101     } catch (Throwable t) {
102       compressionTestResults[algo.ordinal()] = false; // failure
103       throw new DoNotRetryIOException(t);
104     }
105   }
106
107   protected static Path path = new Path(".hfile-comp-test");
108
109   public static void usage() {
110
111     System.err.println(
112       "Usage: CompressionTest <path> " +
113       StringUtils.join( Compression.Algorithm.values(), "|").toLowerCase(Locale.ROOT) +
114       "\n" +
115       "For example:\n" +
116       "  hbase " + CompressionTest.class + " file:///tmp/testfile gz\n");
117     System.exit(1);
118   }
119
120   public static void doSmokeTest(FileSystem fs, Path path, String codec)
121   throws Exception {
122     Configuration conf = HBaseConfiguration.create();
123     HFileContext context = new HFileContextBuilder()
124                            .withCompression(HFileWriterImpl.compressionByName(codec)).build();
125     HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
126         .withPath(fs, path)
127         .withFileContext(context)
128         .create();
129     // Write any-old Cell...
130     final byte [] rowKey = Bytes.toBytes("compressiontestkey");
131     Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval"));
132     writer.append(c);
133     writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval"));
134     writer.close();
135     Cell cc = null;
136     HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
137     try {
138       reader.loadFileInfo();
139       HFileScanner scanner = reader.getScanner(false, true);
140       scanner.seekTo(); // position to the start of file
141       // Scanner does not do Cells yet. Do below for now till fixed.
142       cc = scanner.getCell();
143       if (CellComparator.COMPARATOR.compareRows(c, cc) != 0) {
144         throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString());
145       }
146     } finally {
147       reader.close();
148     }
149   }
150
151   public static void main(String[] args) throws Exception {
152     if (args.length != 2) {
153       usage();
154       System.exit(1);
155     }
156
157     Configuration conf = new Configuration();
158     Path path = new Path(args[0]);
159     FileSystem fs = path.getFileSystem(conf);
160     if (fs.exists(path)) {
161       System.err.println("The specified path exists, aborting!");
162       System.exit(1);
163     }
164
165     try {
166       doSmokeTest(fs, path, args[1]);
167     } finally {
168       fs.delete(path, false);
169     }
170     System.out.println("SUCCESS");
171   }
172 }