001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.crypto;
019
020import static java.lang.String.format;
021
022import java.io.IOException;
023import java.io.InputStream;
024import java.io.OutputStream;
025import java.security.Key;
026import java.security.MessageDigest;
027import java.security.NoSuchAlgorithmException;
028import java.security.spec.InvalidKeySpecException;
029import java.util.Arrays;
030import java.util.Map;
031import java.util.concurrent.ConcurrentHashMap;
032import javax.crypto.SecretKeyFactory;
033import javax.crypto.spec.PBEKeySpec;
034import javax.crypto.spec.SecretKeySpec;
035import org.apache.commons.io.IOUtils;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.HBaseConfiguration;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.io.crypto.aes.AES;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.Pair;
042import org.apache.hadoop.util.ReflectionUtils;
043import org.apache.yetus.audience.InterfaceAudience;
044import org.slf4j.Logger;
045import org.slf4j.LoggerFactory;
046
047/**
048 * A facade for encryption algorithms and related support.
049 */
050@InterfaceAudience.Public
051public final class Encryption {
052
053  private static final Logger LOG = LoggerFactory.getLogger(Encryption.class);
054
055  /**
056   * Configuration key for globally enable / disable column family encryption
057   */
058  public static final String CRYPTO_ENABLED_CONF_KEY = "hbase.crypto.enabled";
059
060  /**
061   * Default value for globally enable / disable column family encryption (set to "true" for
062   * backward compatibility)
063   */
064  public static final boolean CRYPTO_ENABLED_CONF_DEFAULT = true;
065
066  /**
067   * Configuration key for the hash algorithm used for generating key hash in encrypted HFiles. This
068   * is a MessageDigest algorithm identifier string, like "MD5", "SHA-256" or "SHA-384". (default:
069   * "MD5" for backward compatibility reasons)
070   */
071  public static final String CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY = "hbase.crypto.key.hash.algorithm";
072
073  /**
074   * Default hash algorithm used for generating key hash in encrypted HFiles. (we use "MD5" for
075   * backward compatibility reasons)
076   */
077  public static final String CRYPTO_KEY_HASH_ALGORITHM_CONF_DEFAULT = "MD5";
078
079  /**
080   * Configuration key for specifying the behaviour if the configured hash algorithm differs from
081   * the one used for generating key hash in encrypted HFiles currently being read. - "false"
082   * (default): we won't fail but use the hash algorithm stored in the HFile - "true": we throw an
083   * exception (this can be useful if regulations are enforcing the usage of certain algorithms,
084   * e.g. on FIPS compliant clusters)
085   */
086  public static final String CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_KEY =
087    "hbase.crypto.key.hash.algorithm.failOnMismatch";
088
089  /**
090   * Default behaviour is not to fail if the hash algorithm configured differs from the one used in
091   * the HFile. (this is the more fail-safe approach, allowing us to read encrypted HFiles written
092   * using a different encryption key hash algorithm)
093   */
094  public static final boolean CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_DEFAULT = false;
095
096  /**
097   * Crypto context
098   */
099  @InterfaceAudience.Public
100  public static class Context extends org.apache.hadoop.hbase.io.crypto.Context {
101
102    /** The null crypto context */
103    public static final Context NONE = new Context();
104
105    private Context() {
106      super();
107    }
108
109    private Context(Configuration conf) {
110      super(conf);
111    }
112
113    @Override
114    public Context setCipher(Cipher cipher) {
115      super.setCipher(cipher);
116      return this;
117    }
118
119    @Override
120    public Context setKey(Key key) {
121      super.setKey(key);
122      return this;
123    }
124
125    public Context setKey(byte[] key) {
126      super.setKey(new SecretKeySpec(key, getCipher().getName()));
127      return this;
128    }
129  }
130
131  public static Context newContext() {
132    return new Context();
133  }
134
135  public static Context newContext(Configuration conf) {
136    return new Context(conf);
137  }
138
139  // Prevent instantiation
140  private Encryption() {
141    super();
142  }
143
144  /**
145   * Returns true if the column family encryption feature is enabled globally.
146   */
147  public static boolean isEncryptionEnabled(Configuration conf) {
148    return conf.getBoolean(CRYPTO_ENABLED_CONF_KEY, CRYPTO_ENABLED_CONF_DEFAULT);
149  }
150
151  /**
152   * Get an cipher given a name
153   * @param name the cipher name
154   * @return the cipher, or null if a suitable one could not be found
155   */
156  public static Cipher getCipher(Configuration conf, String name) {
157    return getCipherProvider(conf).getCipher(name);
158  }
159
160  /**
161   * Get names of supported encryption algorithms
162   * @return Array of strings, each represents a supported encryption algorithm
163   */
164  public static String[] getSupportedCiphers() {
165    return getSupportedCiphers(HBaseConfiguration.create());
166  }
167
168  /**
169   * Get names of supported encryption algorithms
170   * @return Array of strings, each represents a supported encryption algorithm
171   */
172  public static String[] getSupportedCiphers(Configuration conf) {
173    return getCipherProvider(conf).getSupportedCiphers();
174  }
175
176  /**
177   * Returns the Hash Algorithm defined in the crypto configuration.
178   */
179  public static String getConfiguredHashAlgorithm(Configuration conf) {
180    return conf.getTrimmed(CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY,
181      CRYPTO_KEY_HASH_ALGORITHM_CONF_DEFAULT);
182  }
183
184  /**
185   * Returns the Hash Algorithm mismatch behaviour defined in the crypto configuration.
186   */
187  public static boolean failOnHashAlgorithmMismatch(Configuration conf) {
188    return conf.getBoolean(CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_KEY,
189      CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_DEFAULT);
190  }
191
192  /**
193   * Returns the hash of the supplied argument, using the hash algorithm specified in the given
194   * config.
195   */
196  public static byte[] computeCryptoKeyHash(Configuration conf, byte[] arg) {
197    String algorithm = getConfiguredHashAlgorithm(conf);
198    try {
199      return hashWithAlg(algorithm, arg);
200    } catch (RuntimeException e) {
201      String message = format(
202        "Error in computeCryptoKeyHash (please check your configuration "
203          + "parameter %s and the security provider configuration of the JVM)",
204        CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY);
205      throw new RuntimeException(message, e);
206    }
207  }
208
209  /**
210   * Return the MD5 digest of the concatenation of the supplied arguments.
211   */
212  public static byte[] hash128(String... args) {
213    return hashWithAlg("MD5", Bytes.toByteArrays(args));
214  }
215
216  /**
217   * Return the MD5 digest of the concatenation of the supplied arguments.
218   */
219  public static byte[] hash128(byte[]... args) {
220    return hashWithAlg("MD5", args);
221  }
222
223  /**
224   * Return the SHA-256 digest of the concatenation of the supplied arguments.
225   */
226  public static byte[] hash256(String... args) {
227    return hashWithAlg("SHA-256", Bytes.toByteArrays(args));
228  }
229
230  /**
231   * Return the SHA-256 digest of the concatenation of the supplied arguments.
232   */
233  public static byte[] hash256(byte[]... args) {
234    return hashWithAlg("SHA-256", args);
235  }
236
237  /**
238   * Return a 128 bit key derived from the concatenation of the supplied arguments using
239   * PBKDF2WithHmacSHA1 at 10,000 iterations.
240   */
241  public static byte[] pbkdf128(String... args) {
242    StringBuilder sb = new StringBuilder();
243    for (String s : args) {
244      sb.append(s);
245    }
246    return generateSecretKey("PBKDF2WithHmacSHA1", AES.KEY_LENGTH, sb.toString().toCharArray());
247  }
248
249  /**
250   * Return a 128 bit key derived from the concatenation of the supplied arguments using
251   * PBKDF2WithHmacSHA1 at 10,000 iterations.
252   */
253  public static byte[] pbkdf128(byte[]... args) {
254    StringBuilder sb = new StringBuilder();
255    for (byte[] b : args) {
256      sb.append(Arrays.toString(b));
257    }
258    return generateSecretKey("PBKDF2WithHmacSHA1", AES.KEY_LENGTH, sb.toString().toCharArray());
259  }
260
261  /**
262   * Return a key derived from the concatenation of the supplied arguments using
263   * PBKDF2WithHmacSHA384 key derivation algorithm at 10,000 iterations. The length of the returned
264   * key is determined based on the need of the cypher algorithm. E.g. for the default "AES" we will
265   * need a 128 bit long key, while if the user is using a custom cipher, we might generate keys
266   * with other length. This key generation method is used currently e.g. in the HBase Shell
267   * (admin.rb) to generate a column family data encryption key, if the user provided an
268   * ENCRYPTION_KEY parameter.
269   */
270  public static byte[] generateSecretKey(Configuration conf, String cypherAlg, String... args) {
271    StringBuilder sb = new StringBuilder();
272    for (String s : args) {
273      sb.append(s);
274    }
275    int keyLengthBytes = Encryption.getCipher(conf, cypherAlg).getKeyLength();
276    return generateSecretKey("PBKDF2WithHmacSHA384", keyLengthBytes, sb.toString().toCharArray());
277  }
278
279  /**
280   * Return a key derived from the concatenation of the supplied arguments using
281   * PBKDF2WithHmacSHA384 key derivation algorithm at 10,000 iterations. The length of the returned
282   * key is determined based on the need of the cypher algorithm. E.g. for the default "AES" we will
283   * need a 128 bit long key, while if the user is using a custom cipher, we might generate keys
284   * with other length. This key generation method is used currently e.g. in the HBase Shell
285   * (admin.rb) to generate a column family data encryption key, if the user provided an
286   * ENCRYPTION_KEY parameter.
287   */
288  public static byte[] generateSecretKey(Configuration conf, String cypherAlg, byte[]... args) {
289    StringBuilder sb = new StringBuilder();
290    for (byte[] b : args) {
291      sb.append(Arrays.toString(b));
292    }
293    int keyLength = Encryption.getCipher(conf, cypherAlg).getKeyLength();
294    return generateSecretKey("PBKDF2WithHmacSHA384", keyLength, sb.toString().toCharArray());
295  }
296
297  /**
298   * Return a key (byte array) derived from the supplied password argument using the given algorithm
299   * with a random salt at 10,000 iterations.
300   * @param algorithm      the secret key generation algorithm to use
301   * @param keyLengthBytes the length of the key to be derived (in bytes, not in bits)
302   * @param password       char array to use as password for the key generation algorithm
303   * @return secret key encoded as a byte array
304   */
305  private static byte[] generateSecretKey(String algorithm, int keyLengthBytes, char[] password) {
306    byte[] salt = new byte[keyLengthBytes];
307    Bytes.secureRandom(salt);
308    PBEKeySpec spec = new PBEKeySpec(password, salt, 10000, keyLengthBytes * 8);
309    try {
310      return SecretKeyFactory.getInstance(algorithm).generateSecret(spec).getEncoded();
311    } catch (NoSuchAlgorithmException | InvalidKeySpecException e) {
312      throw new RuntimeException(e);
313    }
314  }
315
316  /**
317   * Encrypt a block of plaintext
318   * <p>
319   * The encryptor's state will be finalized. It should be reinitialized or returned to the pool.
320   * @param out ciphertext
321   * @param src plaintext nnnn
322   */
323  public static void encrypt(OutputStream out, byte[] src, int offset, int length, Encryptor e)
324    throws IOException {
325    OutputStream cout = e.createEncryptionStream(out);
326    try {
327      cout.write(src, offset, length);
328    } finally {
329      cout.close();
330    }
331  }
332
333  /**
334   * Encrypt a block of plaintext
335   * @param out ciphertext
336   * @param src plaintext nnnnn
337   */
338  public static void encrypt(OutputStream out, byte[] src, int offset, int length, Context context,
339    byte[] iv) throws IOException {
340    Encryptor e = context.getCipher().getEncryptor();
341    e.setKey(context.getKey());
342    e.setIv(iv); // can be null
343    e.reset();
344    encrypt(out, src, offset, length, e);
345  }
346
347  /**
348   * Encrypt a stream of plaintext given an encryptor
349   * <p>
350   * The encryptor's state will be finalized. It should be reinitialized or returned to the pool.
351   * @param out ciphertext
352   * @param in  plaintext nn
353   */
354  public static void encrypt(OutputStream out, InputStream in, Encryptor e) throws IOException {
355    OutputStream cout = e.createEncryptionStream(out);
356    try {
357      IOUtils.copy(in, cout);
358    } finally {
359      cout.close();
360    }
361  }
362
363  /**
364   * Encrypt a stream of plaintext given a context and IV
365   * @param out ciphertext
366   * @param in  plaintet nnn
367   */
368  public static void encrypt(OutputStream out, InputStream in, Context context, byte[] iv)
369    throws IOException {
370    Encryptor e = context.getCipher().getEncryptor();
371    e.setKey(context.getKey());
372    e.setIv(iv); // can be null
373    e.reset();
374    encrypt(out, in, e);
375  }
376
377  /**
378   * Decrypt a block of ciphertext read in from a stream with the given cipher and context
379   * <p>
380   * The decryptor's state will be finalized. It should be reinitialized or returned to the pool.
381   * nnnnnn
382   */
383  public static void decrypt(byte[] dest, int destOffset, InputStream in, int destSize, Decryptor d)
384    throws IOException {
385    InputStream cin = d.createDecryptionStream(in);
386    try {
387      IOUtils.readFully(cin, dest, destOffset, destSize);
388    } finally {
389      cin.close();
390    }
391  }
392
393  /**
394   * Decrypt a block of ciphertext from a stream given a context and IV nnnnnnn
395   */
396  public static void decrypt(byte[] dest, int destOffset, InputStream in, int destSize,
397    Context context, byte[] iv) throws IOException {
398    Decryptor d = context.getCipher().getDecryptor();
399    d.setKey(context.getKey());
400    d.setIv(iv); // can be null
401    decrypt(dest, destOffset, in, destSize, d);
402  }
403
404  /**
405   * Decrypt a stream of ciphertext given a decryptor nnnnn
406   */
407  public static void decrypt(OutputStream out, InputStream in, int outLen, Decryptor d)
408    throws IOException {
409    InputStream cin = d.createDecryptionStream(in);
410    byte buf[] = new byte[8 * 1024];
411    long remaining = outLen;
412    try {
413      while (remaining > 0) {
414        int toRead = (int) (remaining < buf.length ? remaining : buf.length);
415        int read = cin.read(buf, 0, toRead);
416        if (read < 0) {
417          break;
418        }
419        out.write(buf, 0, read);
420        remaining -= read;
421      }
422    } finally {
423      cin.close();
424    }
425  }
426
427  /**
428   * Decrypt a stream of ciphertext given a context and IV nnnnnn
429   */
430  public static void decrypt(OutputStream out, InputStream in, int outLen, Context context,
431    byte[] iv) throws IOException {
432    Decryptor d = context.getCipher().getDecryptor();
433    d.setKey(context.getKey());
434    d.setIv(iv); // can be null
435    decrypt(out, in, outLen, d);
436  }
437
438  /**
439   * Resolves a key for the given subject nn * @return a key for the given subject
440   * @throws IOException if the key is not found
441   */
442  public static Key getSecretKeyForSubject(String subject, Configuration conf) throws IOException {
443    KeyProvider provider = getKeyProvider(conf);
444    if (provider != null) {
445      try {
446        Key[] keys = provider.getKeys(new String[] { subject });
447        if (keys != null && keys.length > 0) {
448          return keys[0];
449        }
450      } catch (Exception e) {
451        throw new IOException(e);
452      }
453    }
454    throw new IOException("No key found for subject '" + subject + "'");
455  }
456
457  /**
458   * Encrypts a block of plaintext with the symmetric key resolved for the given subject
459   * @param out    ciphertext
460   * @param in     plaintext
461   * @param conf   configuration
462   * @param cipher the encryption algorithm
463   * @param iv     the initialization vector, can be null n
464   */
465  public static void encryptWithSubjectKey(OutputStream out, InputStream in, String subject,
466    Configuration conf, Cipher cipher, byte[] iv) throws IOException {
467    Key key = getSecretKeyForSubject(subject, conf);
468    if (key == null) {
469      throw new IOException("No key found for subject '" + subject + "'");
470    }
471    Encryptor e = cipher.getEncryptor();
472    e.setKey(key);
473    e.setIv(iv); // can be null
474    encrypt(out, in, e);
475  }
476
477  /**
478   * Decrypts a block of ciphertext with the symmetric key resolved for the given subject
479   * @param out     plaintext
480   * @param in      ciphertext
481   * @param outLen  the expected plaintext length
482   * @param subject the subject's key alias
483   * @param conf    configuration
484   * @param cipher  the encryption algorithm
485   * @param iv      the initialization vector, can be null n
486   */
487  public static void decryptWithSubjectKey(OutputStream out, InputStream in, int outLen,
488    String subject, Configuration conf, Cipher cipher, byte[] iv) throws IOException {
489    Key key = getSecretKeyForSubject(subject, conf);
490    if (key == null) {
491      throw new IOException("No key found for subject '" + subject + "'");
492    }
493    Decryptor d = cipher.getDecryptor();
494    d.setKey(key);
495    d.setIv(iv); // can be null
496    try {
497      decrypt(out, in, outLen, d);
498    } catch (IOException e) {
499      // If the current cipher algorithm fails to unwrap, try the alternate cipher algorithm, if one
500      // is configured
501      String alternateAlgorithm = conf.get(HConstants.CRYPTO_ALTERNATE_KEY_ALGORITHM_CONF_KEY);
502      if (alternateAlgorithm != null) {
503        if (LOG.isDebugEnabled()) {
504          LOG.debug("Unable to decrypt data with current cipher algorithm '"
505            + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES)
506            + "'. Trying with the alternate cipher algorithm '" + alternateAlgorithm
507            + "' configured.");
508        }
509        Cipher alterCipher = Encryption.getCipher(conf, alternateAlgorithm);
510        if (alterCipher == null) {
511          throw new RuntimeException("Cipher '" + alternateAlgorithm + "' not available");
512        }
513        d = alterCipher.getDecryptor();
514        d.setKey(key);
515        d.setIv(iv); // can be null
516        decrypt(out, in, outLen, d);
517      } else {
518        throw new IOException(e);
519      }
520    }
521  }
522
523  private static ClassLoader getClassLoaderForClass(Class<?> c) {
524    ClassLoader cl = Thread.currentThread().getContextClassLoader();
525    if (cl == null) {
526      cl = c.getClassLoader();
527    }
528    if (cl == null) {
529      cl = ClassLoader.getSystemClassLoader();
530    }
531    if (cl == null) {
532      throw new RuntimeException("A ClassLoader to load the Cipher could not be determined");
533    }
534    return cl;
535  }
536
537  public static CipherProvider getCipherProvider(Configuration conf) {
538    String providerClassName =
539      conf.get(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY, DefaultCipherProvider.class.getName());
540    try {
541      CipherProvider provider = (CipherProvider) ReflectionUtils.newInstance(
542        getClassLoaderForClass(CipherProvider.class).loadClass(providerClassName), conf);
543      return provider;
544    } catch (Exception e) {
545      throw new RuntimeException(e);
546    }
547  }
548
549  static final Map<Pair<String, String>, KeyProvider> keyProviderCache = new ConcurrentHashMap<>();
550
551  public static KeyProvider getKeyProvider(Configuration conf) {
552    String providerClassName =
553      conf.get(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyStoreKeyProvider.class.getName());
554    String providerParameters = conf.get(HConstants.CRYPTO_KEYPROVIDER_PARAMETERS_KEY, "");
555    try {
556      Pair<String, String> providerCacheKey = new Pair<>(providerClassName, providerParameters);
557      KeyProvider provider = keyProviderCache.get(providerCacheKey);
558      if (provider != null) {
559        return provider;
560      }
561      provider = (KeyProvider) ReflectionUtils
562        .newInstance(getClassLoaderForClass(KeyProvider.class).loadClass(providerClassName), conf);
563      provider.init(providerParameters);
564      if (LOG.isDebugEnabled()) {
565        LOG.debug("Installed " + providerClassName + " into key provider cache");
566      }
567      keyProviderCache.put(providerCacheKey, provider);
568      return provider;
569    } catch (Exception e) {
570      throw new RuntimeException(e);
571    }
572  }
573
574  public static void incrementIv(byte[] iv) {
575    incrementIv(iv, 1);
576  }
577
578  public static void incrementIv(byte[] iv, int v) {
579    // v should be > 0
580    int length = iv.length;
581    int sum = 0;
582    for (int i = 0; i < length; i++) {
583      if (v <= 0) {
584        break;
585      }
586      sum = v + (iv[i] & 0xFF);
587      v = sum / 256;
588      iv[i] = (byte) (sum % 256);
589    }
590  }
591
592  /**
593   * Return the hash of the concatenation of the supplied arguments, using the hash algorithm
594   * provided.
595   */
596  public static byte[] hashWithAlg(String algorithm, byte[]... args) {
597    try {
598      MessageDigest md = MessageDigest.getInstance(algorithm);
599      for (byte[] arg : args) {
600        md.update(arg);
601      }
602      return md.digest();
603    } catch (NoSuchAlgorithmException e) {
604      throw new RuntimeException("unable to use hash algorithm: " + algorithm, e);
605    }
606  }
607
608}