001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with this
004 * work for additional information regarding copyright ownership. The ASF
005 * licenses this file to you under the Apache License, Version 2.0 (the
006 * "License"); you may not use this file except in compliance with the License.
007 * You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
013 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
014 * License for the specific language governing permissions and limitations under
015 * the License.
016 */
017package org.apache.hadoop.hbase.io.crypto;
018
019import java.io.IOException;
020import java.io.InputStream;
021import java.io.OutputStream;
022import java.security.DigestException;
023import java.security.Key;
024import java.security.MessageDigest;
025import java.security.NoSuchAlgorithmException;
026import java.security.spec.InvalidKeySpecException;
027import java.util.Arrays;
028import java.util.Map;
029import java.util.concurrent.ConcurrentHashMap;
030
031import javax.crypto.SecretKeyFactory;
032import javax.crypto.spec.PBEKeySpec;
033import javax.crypto.spec.SecretKeySpec;
034
035import org.apache.commons.io.IOUtils;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.HBaseConfiguration;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.Pair;
041import org.apache.hadoop.util.ReflectionUtils;
042import org.apache.yetus.audience.InterfaceAudience;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046/**
047 * A facade for encryption algorithms and related support.
048 */
049@InterfaceAudience.Public
050public final class Encryption {
051
052  private static final Logger LOG = LoggerFactory.getLogger(Encryption.class);
053
054  /**
055   * Crypto context
056   */
057  @InterfaceAudience.Public
058  public static class Context extends org.apache.hadoop.hbase.io.crypto.Context {
059
060    /** The null crypto context */
061    public static final Context NONE = new Context();
062
063    private Context() {
064      super();
065    }
066
067    private Context(Configuration conf) {
068      super(conf);
069    }
070
071    @Override
072    public Context setCipher(Cipher cipher) {
073      super.setCipher(cipher);
074      return this;
075    }
076
077    @Override
078    public Context setKey(Key key) {
079      super.setKey(key);
080      return this;
081    }
082
083    public Context setKey(byte[] key) {
084      super.setKey(new SecretKeySpec(key, getCipher().getName()));
085      return this;
086    }
087  }
088
089  public static Context newContext() {
090    return new Context();
091  }
092
093  public static Context newContext(Configuration conf) {
094    return new Context(conf);
095  }
096
097  // Prevent instantiation
098  private Encryption() {
099    super();
100  }
101
102  /**
103   * Get an cipher given a name
104   * @param name the cipher name
105   * @return the cipher, or null if a suitable one could not be found
106   */
107  public static Cipher getCipher(Configuration conf, String name) {
108    return getCipherProvider(conf).getCipher(name);
109  }
110
111  /**
112   * Get names of supported encryption algorithms
113   *
114   * @return Array of strings, each represents a supported encryption algorithm
115   */
116  public static String[] getSupportedCiphers() {
117    return getSupportedCiphers(HBaseConfiguration.create());
118  }
119
120  /**
121   * Get names of supported encryption algorithms
122   *
123   * @return Array of strings, each represents a supported encryption algorithm
124   */
125  public static String[] getSupportedCiphers(Configuration conf) {
126    return getCipherProvider(conf).getSupportedCiphers();
127  }
128
129  /**
130   * Return the MD5 digest of the concatenation of the supplied arguments.
131   */
132  public static byte[] hash128(String... args) {
133    byte[] result = new byte[16];
134    try {
135      MessageDigest md = MessageDigest.getInstance("MD5");
136      for (String arg: args) {
137        md.update(Bytes.toBytes(arg));
138      }
139      md.digest(result, 0, result.length);
140      return result;
141    } catch (NoSuchAlgorithmException e) {
142      throw new RuntimeException(e);
143    } catch (DigestException e) {
144      throw new RuntimeException(e);
145    }
146  }
147
148  /**
149   * Return the MD5 digest of the concatenation of the supplied arguments.
150   */
151  public static byte[] hash128(byte[]... args) {
152    byte[] result = new byte[16];
153    try {
154      MessageDigest md = MessageDigest.getInstance("MD5");
155      for (byte[] arg: args) {
156        md.update(arg);
157      }
158      md.digest(result, 0, result.length);
159      return result;
160    } catch (NoSuchAlgorithmException e) {
161      throw new RuntimeException(e);
162    } catch (DigestException e) {
163      throw new RuntimeException(e);
164    }
165  }
166
167  /**
168   * Return the SHA-256 digest of the concatenation of the supplied arguments.
169   */
170  public static byte[] hash256(String... args) {
171    byte[] result = new byte[32];
172    try {
173      MessageDigest md = MessageDigest.getInstance("SHA-256");
174      for (String arg: args) {
175        md.update(Bytes.toBytes(arg));
176      }
177      md.digest(result, 0, result.length);
178      return result;
179    } catch (NoSuchAlgorithmException e) {
180      throw new RuntimeException(e);
181    } catch (DigestException e) {
182      throw new RuntimeException(e);
183    }
184  }
185
186  /**
187   * Return the SHA-256 digest of the concatenation of the supplied arguments.
188   */
189  public static byte[] hash256(byte[]... args) {
190    byte[] result = new byte[32];
191    try {
192      MessageDigest md = MessageDigest.getInstance("SHA-256");
193      for (byte[] arg: args) {
194        md.update(arg);
195      }
196      md.digest(result, 0, result.length);
197      return result;
198    } catch (NoSuchAlgorithmException e) {
199      throw new RuntimeException(e);
200    } catch (DigestException e) {
201      throw new RuntimeException(e);
202    }
203  }
204
205  /**
206   * Return a 128 bit key derived from the concatenation of the supplied
207   * arguments using PBKDF2WithHmacSHA1 at 10,000 iterations.
208   *
209   */
210  public static byte[] pbkdf128(String... args) {
211    byte[] salt = new byte[128];
212    Bytes.random(salt);
213    StringBuilder sb = new StringBuilder();
214    for (String s: args) {
215      sb.append(s);
216    }
217    PBEKeySpec spec = new PBEKeySpec(sb.toString().toCharArray(), salt, 10000, 128);
218    try {
219      return SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1")
220        .generateSecret(spec).getEncoded();
221    } catch (NoSuchAlgorithmException e) {
222      throw new RuntimeException(e);
223    } catch (InvalidKeySpecException e) {
224      throw new RuntimeException(e);
225    }
226  }
227
228  /**
229   * Return a 128 bit key derived from the concatenation of the supplied
230   * arguments using PBKDF2WithHmacSHA1 at 10,000 iterations.
231   *
232   */
233  public static byte[] pbkdf128(byte[]... args) {
234    byte[] salt = new byte[128];
235    Bytes.random(salt);
236    StringBuilder sb = new StringBuilder();
237    for (byte[] b: args) {
238      sb.append(Arrays.toString(b));
239    }
240    PBEKeySpec spec = new PBEKeySpec(sb.toString().toCharArray(), salt, 10000, 128);
241    try {
242      return SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1")
243        .generateSecret(spec).getEncoded();
244    } catch (NoSuchAlgorithmException e) {
245      throw new RuntimeException(e);
246    } catch (InvalidKeySpecException e) {
247      throw new RuntimeException(e);
248    }
249  }
250
251  /**
252   * Encrypt a block of plaintext
253   * <p>
254   * The encryptor's state will be finalized. It should be reinitialized or
255   * returned to the pool.
256   * @param out ciphertext
257   * @param src plaintext
258   * @param offset
259   * @param length
260   * @param e
261   * @throws IOException
262    */
263  public static void encrypt(OutputStream out, byte[] src, int offset,
264      int length, Encryptor e) throws IOException {
265    OutputStream cout = e.createEncryptionStream(out);
266    try {
267      cout.write(src, offset, length);
268    } finally {
269      cout.close();
270    }
271  }
272
273  /**
274   * Encrypt a block of plaintext
275   * @param out ciphertext
276   * @param src plaintext
277   * @param offset
278   * @param length
279   * @param context
280   * @param iv
281   * @throws IOException
282    */
283  public static void encrypt(OutputStream out, byte[] src, int offset,
284      int length, Context context, byte[] iv) throws IOException {
285    Encryptor e = context.getCipher().getEncryptor();
286    e.setKey(context.getKey());
287    e.setIv(iv); // can be null
288    e.reset();
289    encrypt(out, src, offset, length, e);
290  }
291
292  /**
293   * Encrypt a stream of plaintext given an encryptor
294   * <p>
295   * The encryptor's state will be finalized. It should be reinitialized or
296   * returned to the pool.
297   * @param out ciphertext
298   * @param in plaintext
299   * @param e
300   * @throws IOException
301   */
302  public static void encrypt(OutputStream out, InputStream in, Encryptor e)
303      throws IOException {
304    OutputStream cout = e.createEncryptionStream(out);
305    try {
306      IOUtils.copy(in, cout);
307    } finally {
308      cout.close();
309    }
310  }
311
312  /**
313   * Encrypt a stream of plaintext given a context and IV
314   * @param out ciphertext
315   * @param in plaintet
316   * @param context
317   * @param iv
318   * @throws IOException
319   */
320  public static void encrypt(OutputStream out, InputStream in, Context context,
321      byte[] iv) throws IOException {
322    Encryptor e = context.getCipher().getEncryptor();
323    e.setKey(context.getKey());
324    e.setIv(iv); // can be null
325    e.reset();
326    encrypt(out, in, e);
327  }
328
329  /**
330   * Decrypt a block of ciphertext read in from a stream with the given
331   * cipher and context
332   * <p>
333   * The decryptor's state will be finalized. It should be reinitialized or
334   * returned to the pool.
335   * @param dest
336   * @param destOffset
337   * @param in
338   * @param destSize
339   * @param d
340   * @throws IOException
341   */
342  public static void decrypt(byte[] dest, int destOffset, InputStream in,
343      int destSize, Decryptor d) throws IOException {
344    InputStream cin = d.createDecryptionStream(in);
345    try {
346      IOUtils.readFully(cin, dest, destOffset, destSize);
347    } finally {
348      cin.close();
349    }
350  }
351
352  /**
353   * Decrypt a block of ciphertext from a stream given a context and IV
354   * @param dest
355   * @param destOffset
356   * @param in
357   * @param destSize
358   * @param context
359   * @param iv
360   * @throws IOException
361   */
362  public static void decrypt(byte[] dest, int destOffset, InputStream in,
363      int destSize, Context context, byte[] iv) throws IOException {
364    Decryptor d = context.getCipher().getDecryptor();
365    d.setKey(context.getKey());
366    d.setIv(iv); // can be null
367    decrypt(dest, destOffset, in, destSize, d);
368  }
369
370  /**
371   * Decrypt a stream of ciphertext given a decryptor
372   * @param out
373   * @param in
374   * @param outLen
375   * @param d
376   * @throws IOException
377   */
378  public static void decrypt(OutputStream out, InputStream in, int outLen,
379      Decryptor d) throws IOException {
380    InputStream cin = d.createDecryptionStream(in);
381    byte buf[] = new byte[8*1024];
382    long remaining = outLen;
383    try {
384      while (remaining > 0) {
385        int toRead = (int)(remaining < buf.length ? remaining : buf.length);
386        int read = cin.read(buf, 0, toRead);
387        if (read < 0) {
388          break;
389        }
390        out.write(buf, 0, read);
391        remaining -= read;
392      }
393    } finally {
394      cin.close();
395    }
396  }
397
398  /**
399   * Decrypt a stream of ciphertext given a context and IV
400   * @param out
401   * @param in
402   * @param outLen
403   * @param context
404   * @param iv
405   * @throws IOException
406   */
407  public static void decrypt(OutputStream out, InputStream in, int outLen,
408      Context context, byte[] iv) throws IOException {
409    Decryptor d = context.getCipher().getDecryptor();
410    d.setKey(context.getKey());
411    d.setIv(iv); // can be null
412    decrypt(out, in, outLen, d);
413  }
414
415  /**
416   * Resolves a key for the given subject
417   * @param subject
418   * @param conf
419   * @return a key for the given subject
420   * @throws IOException if the key is not found
421   */
422  public static Key getSecretKeyForSubject(String subject, Configuration conf)
423      throws IOException {
424    KeyProvider provider = getKeyProvider(conf);
425    if (provider != null) try {
426      Key[] keys = provider.getKeys(new String[] { subject });
427      if (keys != null && keys.length > 0) {
428        return keys[0];
429      }
430    } catch (Exception e) {
431      throw new IOException(e);
432    }
433    throw new IOException("No key found for subject '" + subject + "'");
434  }
435
436  /**
437   * Encrypts a block of plaintext with the symmetric key resolved for the given subject
438   * @param out ciphertext
439   * @param in plaintext
440   * @param conf configuration
441   * @param cipher the encryption algorithm
442   * @param iv the initialization vector, can be null
443   * @throws IOException
444   */
445  public static void encryptWithSubjectKey(OutputStream out, InputStream in,
446      String subject, Configuration conf, Cipher cipher, byte[] iv)
447      throws IOException {
448    Key key = getSecretKeyForSubject(subject, conf);
449    if (key == null) {
450      throw new IOException("No key found for subject '" + subject + "'");
451    }
452    Encryptor e = cipher.getEncryptor();
453    e.setKey(key);
454    e.setIv(iv); // can be null
455    encrypt(out, in, e);
456  }
457
458  /**
459   * Decrypts a block of ciphertext with the symmetric key resolved for the given subject
460   * @param out plaintext
461   * @param in ciphertext
462   * @param outLen the expected plaintext length
463   * @param subject the subject's key alias
464   * @param conf configuration
465   * @param cipher the encryption algorithm
466   * @param iv the initialization vector, can be null
467   * @throws IOException
468   */
469  public static void decryptWithSubjectKey(OutputStream out, InputStream in, int outLen,
470      String subject, Configuration conf, Cipher cipher, byte[] iv) throws IOException {
471    Key key = getSecretKeyForSubject(subject, conf);
472    if (key == null) {
473      throw new IOException("No key found for subject '" + subject + "'");
474    }
475    Decryptor d = cipher.getDecryptor();
476    d.setKey(key);
477    d.setIv(iv); // can be null
478    try {
479      decrypt(out, in, outLen, d);
480    } catch (IOException e) {
481      // If the current cipher algorithm fails to unwrap, try the alternate cipher algorithm, if one
482      // is configured
483      String alternateAlgorithm = conf.get(HConstants.CRYPTO_ALTERNATE_KEY_ALGORITHM_CONF_KEY);
484      if (alternateAlgorithm != null) {
485        if (LOG.isDebugEnabled()) {
486          LOG.debug("Unable to decrypt data with current cipher algorithm '"
487              + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES)
488              + "'. Trying with the alternate cipher algorithm '" + alternateAlgorithm
489              + "' configured.");
490        }
491        Cipher alterCipher = Encryption.getCipher(conf, alternateAlgorithm);
492        if (alterCipher == null) {
493          throw new RuntimeException("Cipher '" + alternateAlgorithm + "' not available");
494        }
495        d = alterCipher.getDecryptor();
496        d.setKey(key);
497        d.setIv(iv); // can be null
498        decrypt(out, in, outLen, d);
499      } else {
500        throw new IOException(e);
501      }
502    }
503  }
504
505  private static ClassLoader getClassLoaderForClass(Class<?> c) {
506    ClassLoader cl = Thread.currentThread().getContextClassLoader();
507    if (cl == null) {
508      cl = c.getClassLoader();
509    }
510    if (cl == null) {
511      cl = ClassLoader.getSystemClassLoader();
512    }
513    if (cl == null) {
514      throw new RuntimeException("A ClassLoader to load the Cipher could not be determined");
515    }
516    return cl;
517  }
518
519  public static CipherProvider getCipherProvider(Configuration conf) {
520    String providerClassName = conf.get(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY,
521      DefaultCipherProvider.class.getName());
522    try {
523      CipherProvider provider = (CipherProvider)
524        ReflectionUtils.newInstance(getClassLoaderForClass(CipherProvider.class)
525          .loadClass(providerClassName),
526        conf);
527      return provider;
528    } catch (Exception e) {
529      throw new RuntimeException(e);
530    }
531  }
532
533  static final Map<Pair<String,String>,KeyProvider> keyProviderCache = new ConcurrentHashMap<>();
534
535  public static KeyProvider getKeyProvider(Configuration conf) {
536    String providerClassName = conf.get(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY,
537      KeyStoreKeyProvider.class.getName());
538    String providerParameters = conf.get(HConstants.CRYPTO_KEYPROVIDER_PARAMETERS_KEY, "");
539    try {
540      Pair<String,String> providerCacheKey = new Pair<>(providerClassName,
541        providerParameters);
542      KeyProvider provider = keyProviderCache.get(providerCacheKey);
543      if (provider != null) {
544        return provider;
545      }
546      provider = (KeyProvider) ReflectionUtils.newInstance(
547        getClassLoaderForClass(KeyProvider.class).loadClass(providerClassName),
548        conf);
549      provider.init(providerParameters);
550      if (LOG.isDebugEnabled()) {
551        LOG.debug("Installed " + providerClassName + " into key provider cache");
552      }
553      keyProviderCache.put(providerCacheKey, provider);
554      return provider;
555    } catch (Exception e) {
556      throw new RuntimeException(e);
557    }
558  }
559
560  public static void incrementIv(byte[] iv) {
561    incrementIv(iv, 1);
562  }
563
564  public static void incrementIv(byte[] iv, int v) {
565    int length = iv.length;
566    boolean carry = true;
567    // TODO: Optimize for v > 1, e.g. 16, 32
568    do {
569      for (int i = 0; i < length; i++) {
570        if (carry) {
571          iv[i] = (byte) ((iv[i] + 1) & 0xFF);
572          carry = 0 == iv[i];
573        } else {
574          break;
575        }
576      }
577      v--;
578    } while (v > 0);
579  }
580
581}