001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.crypto; 019 020import static java.lang.String.format; 021 022import java.io.IOException; 023import java.io.InputStream; 024import java.io.OutputStream; 025import java.security.Key; 026import java.security.MessageDigest; 027import java.security.NoSuchAlgorithmException; 028import java.security.spec.InvalidKeySpecException; 029import java.util.Arrays; 030import java.util.Map; 031import java.util.concurrent.ConcurrentHashMap; 032import javax.crypto.SecretKeyFactory; 033import javax.crypto.spec.PBEKeySpec; 034import javax.crypto.spec.SecretKeySpec; 035import org.apache.commons.io.IOUtils; 036import org.apache.hadoop.conf.Configuration; 037import org.apache.hadoop.hbase.HBaseConfiguration; 038import org.apache.hadoop.hbase.HConstants; 039import org.apache.hadoop.hbase.io.crypto.aes.AES; 040import org.apache.hadoop.hbase.util.Bytes; 041import org.apache.hadoop.hbase.util.Pair; 042import org.apache.hadoop.util.ReflectionUtils; 043import org.apache.yetus.audience.InterfaceAudience; 044import org.slf4j.Logger; 045import org.slf4j.LoggerFactory; 046 047/** 048 * A facade for encryption algorithms and related support. 049 */ 050@InterfaceAudience.Public 051public final class Encryption { 052 053 private static final Logger LOG = LoggerFactory.getLogger(Encryption.class); 054 055 /** 056 * Configuration key for globally enable / disable column family encryption 057 */ 058 public static final String CRYPTO_ENABLED_CONF_KEY = "hbase.crypto.enabled"; 059 060 /** 061 * Default value for globally enable / disable column family encryption (set to "true" for 062 * backward compatibility) 063 */ 064 public static final boolean CRYPTO_ENABLED_CONF_DEFAULT = true; 065 066 /** 067 * Configuration key for the hash algorithm used for generating key hash in encrypted HFiles. This 068 * is a MessageDigest algorithm identifier string, like "MD5", "SHA-256" or "SHA-384". (default: 069 * "MD5" for backward compatibility reasons) 070 */ 071 public static final String CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY = "hbase.crypto.key.hash.algorithm"; 072 073 /** 074 * Default hash algorithm used for generating key hash in encrypted HFiles. (we use "MD5" for 075 * backward compatibility reasons) 076 */ 077 public static final String CRYPTO_KEY_HASH_ALGORITHM_CONF_DEFAULT = "MD5"; 078 079 /** 080 * Configuration key for specifying the behaviour if the configured hash algorithm differs from 081 * the one used for generating key hash in encrypted HFiles currently being read. - "false" 082 * (default): we won't fail but use the hash algorithm stored in the HFile - "true": we throw an 083 * exception (this can be useful if regulations are enforcing the usage of certain algorithms, 084 * e.g. on FIPS compliant clusters) 085 */ 086 public static final String CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_KEY = 087 "hbase.crypto.key.hash.algorithm.failOnMismatch"; 088 089 /** 090 * Default behaviour is not to fail if the hash algorithm configured differs from the one used in 091 * the HFile. (this is the more fail-safe approach, allowing us to read encrypted HFiles written 092 * using a different encryption key hash algorithm) 093 */ 094 public static final boolean CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_DEFAULT = false; 095 096 /** 097 * Crypto context 098 */ 099 @InterfaceAudience.Public 100 public static class Context extends org.apache.hadoop.hbase.io.crypto.Context { 101 102 /** The null crypto context */ 103 public static final Context NONE = new Context(); 104 105 private Context() { 106 super(); 107 } 108 109 private Context(Configuration conf) { 110 super(conf); 111 } 112 113 @Override 114 public Context setCipher(Cipher cipher) { 115 super.setCipher(cipher); 116 return this; 117 } 118 119 @Override 120 public Context setKey(Key key) { 121 super.setKey(key); 122 return this; 123 } 124 125 public Context setKey(byte[] key) { 126 super.setKey(new SecretKeySpec(key, getCipher().getName())); 127 return this; 128 } 129 } 130 131 public static Context newContext() { 132 return new Context(); 133 } 134 135 public static Context newContext(Configuration conf) { 136 return new Context(conf); 137 } 138 139 // Prevent instantiation 140 private Encryption() { 141 super(); 142 } 143 144 /** 145 * Returns true if the column family encryption feature is enabled globally. 146 */ 147 public static boolean isEncryptionEnabled(Configuration conf) { 148 return conf.getBoolean(CRYPTO_ENABLED_CONF_KEY, CRYPTO_ENABLED_CONF_DEFAULT); 149 } 150 151 /** 152 * Get an cipher given a name 153 * @param name the cipher name 154 * @return the cipher, or null if a suitable one could not be found 155 */ 156 public static Cipher getCipher(Configuration conf, String name) { 157 return getCipherProvider(conf).getCipher(name); 158 } 159 160 /** 161 * Get names of supported encryption algorithms 162 * @return Array of strings, each represents a supported encryption algorithm 163 */ 164 public static String[] getSupportedCiphers() { 165 return getSupportedCiphers(HBaseConfiguration.create()); 166 } 167 168 /** 169 * Get names of supported encryption algorithms 170 * @return Array of strings, each represents a supported encryption algorithm 171 */ 172 public static String[] getSupportedCiphers(Configuration conf) { 173 return getCipherProvider(conf).getSupportedCiphers(); 174 } 175 176 /** 177 * Returns the Hash Algorithm defined in the crypto configuration. 178 */ 179 public static String getConfiguredHashAlgorithm(Configuration conf) { 180 return conf.getTrimmed(CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY, 181 CRYPTO_KEY_HASH_ALGORITHM_CONF_DEFAULT); 182 } 183 184 /** 185 * Returns the Hash Algorithm mismatch behaviour defined in the crypto configuration. 186 */ 187 public static boolean failOnHashAlgorithmMismatch(Configuration conf) { 188 return conf.getBoolean(CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_KEY, 189 CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_DEFAULT); 190 } 191 192 /** 193 * Returns the hash of the supplied argument, using the hash algorithm specified in the given 194 * config. 195 */ 196 public static byte[] computeCryptoKeyHash(Configuration conf, byte[] arg) { 197 String algorithm = getConfiguredHashAlgorithm(conf); 198 try { 199 return hashWithAlg(algorithm, arg); 200 } catch (RuntimeException e) { 201 String message = format( 202 "Error in computeCryptoKeyHash (please check your configuration " 203 + "parameter %s and the security provider configuration of the JVM)", 204 CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY); 205 throw new RuntimeException(message, e); 206 } 207 } 208 209 /** 210 * Return the MD5 digest of the concatenation of the supplied arguments. 211 */ 212 public static byte[] hash128(String... args) { 213 return hashWithAlg("MD5", Bytes.toByteArrays(args)); 214 } 215 216 /** 217 * Return the MD5 digest of the concatenation of the supplied arguments. 218 */ 219 public static byte[] hash128(byte[]... args) { 220 return hashWithAlg("MD5", args); 221 } 222 223 /** 224 * Return the SHA-256 digest of the concatenation of the supplied arguments. 225 */ 226 public static byte[] hash256(String... args) { 227 return hashWithAlg("SHA-256", Bytes.toByteArrays(args)); 228 } 229 230 /** 231 * Return the SHA-256 digest of the concatenation of the supplied arguments. 232 */ 233 public static byte[] hash256(byte[]... args) { 234 return hashWithAlg("SHA-256", args); 235 } 236 237 /** 238 * Return a 128 bit key derived from the concatenation of the supplied arguments using 239 * PBKDF2WithHmacSHA1 at 10,000 iterations. 240 */ 241 public static byte[] pbkdf128(String... args) { 242 StringBuilder sb = new StringBuilder(); 243 for (String s : args) { 244 sb.append(s); 245 } 246 return generateSecretKey("PBKDF2WithHmacSHA1", AES.KEY_LENGTH, sb.toString().toCharArray()); 247 } 248 249 /** 250 * Return a 128 bit key derived from the concatenation of the supplied arguments using 251 * PBKDF2WithHmacSHA1 at 10,000 iterations. 252 */ 253 public static byte[] pbkdf128(byte[]... args) { 254 StringBuilder sb = new StringBuilder(); 255 for (byte[] b : args) { 256 sb.append(Arrays.toString(b)); 257 } 258 return generateSecretKey("PBKDF2WithHmacSHA1", AES.KEY_LENGTH, sb.toString().toCharArray()); 259 } 260 261 /** 262 * Return a key derived from the concatenation of the supplied arguments using 263 * PBKDF2WithHmacSHA384 key derivation algorithm at 10,000 iterations. The length of the returned 264 * key is determined based on the need of the cypher algorithm. E.g. for the default "AES" we will 265 * need a 128 bit long key, while if the user is using a custom cipher, we might generate keys 266 * with other length. This key generation method is used currently e.g. in the HBase Shell 267 * (admin.rb) to generate a column family data encryption key, if the user provided an 268 * ENCRYPTION_KEY parameter. 269 */ 270 public static byte[] generateSecretKey(Configuration conf, String cypherAlg, String... args) { 271 StringBuilder sb = new StringBuilder(); 272 for (String s : args) { 273 sb.append(s); 274 } 275 int keyLengthBytes = Encryption.getCipher(conf, cypherAlg).getKeyLength(); 276 return generateSecretKey("PBKDF2WithHmacSHA384", keyLengthBytes, sb.toString().toCharArray()); 277 } 278 279 /** 280 * Return a key derived from the concatenation of the supplied arguments using 281 * PBKDF2WithHmacSHA384 key derivation algorithm at 10,000 iterations. The length of the returned 282 * key is determined based on the need of the cypher algorithm. E.g. for the default "AES" we will 283 * need a 128 bit long key, while if the user is using a custom cipher, we might generate keys 284 * with other length. This key generation method is used currently e.g. in the HBase Shell 285 * (admin.rb) to generate a column family data encryption key, if the user provided an 286 * ENCRYPTION_KEY parameter. 287 */ 288 public static byte[] generateSecretKey(Configuration conf, String cypherAlg, byte[]... args) { 289 StringBuilder sb = new StringBuilder(); 290 for (byte[] b : args) { 291 sb.append(Arrays.toString(b)); 292 } 293 int keyLength = Encryption.getCipher(conf, cypherAlg).getKeyLength(); 294 return generateSecretKey("PBKDF2WithHmacSHA384", keyLength, sb.toString().toCharArray()); 295 } 296 297 /** 298 * Return a key (byte array) derived from the supplied password argument using the given algorithm 299 * with a random salt at 10,000 iterations. 300 * @param algorithm the secret key generation algorithm to use 301 * @param keyLengthBytes the length of the key to be derived (in bytes, not in bits) 302 * @param password char array to use as password for the key generation algorithm 303 * @return secret key encoded as a byte array 304 */ 305 private static byte[] generateSecretKey(String algorithm, int keyLengthBytes, char[] password) { 306 byte[] salt = new byte[keyLengthBytes]; 307 Bytes.secureRandom(salt); 308 PBEKeySpec spec = new PBEKeySpec(password, salt, 10000, keyLengthBytes * 8); 309 try { 310 return SecretKeyFactory.getInstance(algorithm).generateSecret(spec).getEncoded(); 311 } catch (NoSuchAlgorithmException | InvalidKeySpecException e) { 312 throw new RuntimeException(e); 313 } 314 } 315 316 /** 317 * Encrypt a block of plaintext 318 * <p> 319 * The encryptor's state will be finalized. It should be reinitialized or returned to the pool. 320 * @param out ciphertext 321 * @param src plaintext 322 */ 323 public static void encrypt(OutputStream out, byte[] src, int offset, int length, Encryptor e) 324 throws IOException { 325 OutputStream cout = e.createEncryptionStream(out); 326 try { 327 cout.write(src, offset, length); 328 } finally { 329 cout.close(); 330 } 331 } 332 333 /** 334 * Encrypt a block of plaintext 335 * @param out ciphertext 336 * @param src plaintext 337 */ 338 public static void encrypt(OutputStream out, byte[] src, int offset, int length, Context context, 339 byte[] iv) throws IOException { 340 Encryptor e = context.getCipher().getEncryptor(); 341 e.setKey(context.getKey()); 342 e.setIv(iv); // can be null 343 e.reset(); 344 encrypt(out, src, offset, length, e); 345 } 346 347 /** 348 * Encrypt a stream of plaintext given an encryptor 349 * <p> 350 * The encryptor's state will be finalized. It should be reinitialized or returned to the pool. 351 * @param out ciphertext 352 * @param in plaintext 353 */ 354 public static void encrypt(OutputStream out, InputStream in, Encryptor e) throws IOException { 355 OutputStream cout = e.createEncryptionStream(out); 356 try { 357 IOUtils.copy(in, cout); 358 } finally { 359 cout.close(); 360 } 361 } 362 363 /** 364 * Encrypt a stream of plaintext given a context and IV 365 * @param out ciphertext 366 * @param in plaintet 367 */ 368 public static void encrypt(OutputStream out, InputStream in, Context context, byte[] iv) 369 throws IOException { 370 Encryptor e = context.getCipher().getEncryptor(); 371 e.setKey(context.getKey()); 372 e.setIv(iv); // can be null 373 e.reset(); 374 encrypt(out, in, e); 375 } 376 377 /** 378 * Decrypt a block of ciphertext read in from a stream with the given cipher and context 379 * <p> 380 * The decryptor's state will be finalized. It should be reinitialized or returned to the pool. 381 */ 382 public static void decrypt(byte[] dest, int destOffset, InputStream in, int destSize, Decryptor d) 383 throws IOException { 384 InputStream cin = d.createDecryptionStream(in); 385 try { 386 IOUtils.readFully(cin, dest, destOffset, destSize); 387 } finally { 388 cin.close(); 389 } 390 } 391 392 /** 393 * Decrypt a block of ciphertext from a stream given a context and IV 394 */ 395 public static void decrypt(byte[] dest, int destOffset, InputStream in, int destSize, 396 Context context, byte[] iv) throws IOException { 397 Decryptor d = context.getCipher().getDecryptor(); 398 d.setKey(context.getKey()); 399 d.setIv(iv); // can be null 400 decrypt(dest, destOffset, in, destSize, d); 401 } 402 403 /** 404 * Decrypt a stream of ciphertext given a decryptor 405 */ 406 public static void decrypt(OutputStream out, InputStream in, int outLen, Decryptor d) 407 throws IOException { 408 InputStream cin = d.createDecryptionStream(in); 409 byte buf[] = new byte[8 * 1024]; 410 long remaining = outLen; 411 try { 412 while (remaining > 0) { 413 int toRead = (int) (remaining < buf.length ? remaining : buf.length); 414 int read = cin.read(buf, 0, toRead); 415 if (read < 0) { 416 break; 417 } 418 out.write(buf, 0, read); 419 remaining -= read; 420 } 421 } finally { 422 cin.close(); 423 } 424 } 425 426 /** 427 * Decrypt a stream of ciphertext given a context and IV 428 */ 429 public static void decrypt(OutputStream out, InputStream in, int outLen, Context context, 430 byte[] iv) throws IOException { 431 Decryptor d = context.getCipher().getDecryptor(); 432 d.setKey(context.getKey()); 433 d.setIv(iv); // can be null 434 decrypt(out, in, outLen, d); 435 } 436 437 /** 438 * Resolves a key for the given subject 439 * @return a key for the given subject 440 * @throws IOException if the key is not found 441 */ 442 public static Key getSecretKeyForSubject(String subject, Configuration conf) throws IOException { 443 KeyProvider provider = getKeyProvider(conf); 444 if (provider != null) { 445 try { 446 Key[] keys = provider.getKeys(new String[] { subject }); 447 if (keys != null && keys.length > 0) { 448 return keys[0]; 449 } 450 } catch (Exception e) { 451 throw new IOException(e); 452 } 453 } 454 throw new IOException("No key found for subject '" + subject + "'"); 455 } 456 457 /** 458 * Encrypts a block of plaintext with the symmetric key resolved for the given subject 459 * @param out ciphertext 460 * @param in plaintext 461 * @param conf configuration 462 * @param cipher the encryption algorithm 463 * @param iv the initialization vector, can be null 464 */ 465 public static void encryptWithSubjectKey(OutputStream out, InputStream in, String subject, 466 Configuration conf, Cipher cipher, byte[] iv) throws IOException { 467 Key key = getSecretKeyForSubject(subject, conf); 468 if (key == null) { 469 throw new IOException("No key found for subject '" + subject + "'"); 470 } 471 Encryptor e = cipher.getEncryptor(); 472 e.setKey(key); 473 e.setIv(iv); // can be null 474 encrypt(out, in, e); 475 } 476 477 /** 478 * Decrypts a block of ciphertext with the symmetric key resolved for the given subject 479 * @param out plaintext 480 * @param in ciphertext 481 * @param outLen the expected plaintext length 482 * @param subject the subject's key alias 483 * @param conf configuration 484 * @param cipher the encryption algorithm 485 * @param iv the initialization vector, can be null 486 */ 487 public static void decryptWithSubjectKey(OutputStream out, InputStream in, int outLen, 488 String subject, Configuration conf, Cipher cipher, byte[] iv) throws IOException { 489 Key key = getSecretKeyForSubject(subject, conf); 490 if (key == null) { 491 throw new IOException("No key found for subject '" + subject + "'"); 492 } 493 Decryptor d = cipher.getDecryptor(); 494 d.setKey(key); 495 d.setIv(iv); // can be null 496 try { 497 decrypt(out, in, outLen, d); 498 } catch (IOException e) { 499 // If the current cipher algorithm fails to unwrap, try the alternate cipher algorithm, if one 500 // is configured 501 String alternateAlgorithm = conf.get(HConstants.CRYPTO_ALTERNATE_KEY_ALGORITHM_CONF_KEY); 502 if (alternateAlgorithm != null) { 503 if (LOG.isDebugEnabled()) { 504 LOG.debug("Unable to decrypt data with current cipher algorithm '" 505 + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES) 506 + "'. Trying with the alternate cipher algorithm '" + alternateAlgorithm 507 + "' configured."); 508 } 509 Cipher alterCipher = Encryption.getCipher(conf, alternateAlgorithm); 510 if (alterCipher == null) { 511 throw new RuntimeException("Cipher '" + alternateAlgorithm + "' not available"); 512 } 513 d = alterCipher.getDecryptor(); 514 d.setKey(key); 515 d.setIv(iv); // can be null 516 decrypt(out, in, outLen, d); 517 } else { 518 throw new IOException(e); 519 } 520 } 521 } 522 523 private static ClassLoader getClassLoaderForClass(Class<?> c) { 524 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 525 if (cl == null) { 526 cl = c.getClassLoader(); 527 } 528 if (cl == null) { 529 cl = ClassLoader.getSystemClassLoader(); 530 } 531 if (cl == null) { 532 throw new RuntimeException("A ClassLoader to load the Cipher could not be determined"); 533 } 534 return cl; 535 } 536 537 public static CipherProvider getCipherProvider(Configuration conf) { 538 String providerClassName = 539 conf.get(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY, DefaultCipherProvider.class.getName()); 540 try { 541 CipherProvider provider = (CipherProvider) ReflectionUtils.newInstance( 542 getClassLoaderForClass(CipherProvider.class).loadClass(providerClassName), conf); 543 return provider; 544 } catch (Exception e) { 545 throw new RuntimeException(e); 546 } 547 } 548 549 static final Map<Pair<String, String>, KeyProvider> keyProviderCache = new ConcurrentHashMap<>(); 550 551 public static KeyProvider getKeyProvider(Configuration conf) { 552 String providerClassName = 553 conf.get(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyStoreKeyProvider.class.getName()); 554 String providerParameters = conf.get(HConstants.CRYPTO_KEYPROVIDER_PARAMETERS_KEY, ""); 555 try { 556 Pair<String, String> providerCacheKey = new Pair<>(providerClassName, providerParameters); 557 KeyProvider provider = keyProviderCache.get(providerCacheKey); 558 if (provider != null) { 559 return provider; 560 } 561 provider = (KeyProvider) ReflectionUtils 562 .newInstance(getClassLoaderForClass(KeyProvider.class).loadClass(providerClassName), conf); 563 provider.init(providerParameters); 564 if (LOG.isDebugEnabled()) { 565 LOG.debug("Installed " + providerClassName + " into key provider cache"); 566 } 567 keyProviderCache.put(providerCacheKey, provider); 568 return provider; 569 } catch (Exception e) { 570 throw new RuntimeException(e); 571 } 572 } 573 574 public static void incrementIv(byte[] iv) { 575 incrementIv(iv, 1); 576 } 577 578 public static void incrementIv(byte[] iv, int v) { 579 // v should be > 0 580 int length = iv.length; 581 int sum = 0; 582 for (int i = 0; i < length; i++) { 583 if (v <= 0) { 584 break; 585 } 586 sum = v + (iv[i] & 0xFF); 587 v = sum / 256; 588 iv[i] = (byte) (sum % 256); 589 } 590 } 591 592 /** 593 * Return the hash of the concatenation of the supplied arguments, using the hash algorithm 594 * provided. 595 */ 596 public static byte[] hashWithAlg(String algorithm, byte[]... args) { 597 try { 598 MessageDigest md = MessageDigest.getInstance(algorithm); 599 for (byte[] arg : args) { 600 md.update(arg); 601 } 602 return md.digest(); 603 } catch (NoSuchAlgorithmException e) { 604 throw new RuntimeException("unable to use hash algorithm: " + algorithm, e); 605 } 606 } 607 608}