001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.crypto.tls; 019 020import java.io.IOException; 021import java.nio.file.Path; 022import java.nio.file.Paths; 023import java.security.GeneralSecurityException; 024import java.security.KeyStore; 025import java.security.Security; 026import java.security.cert.PKIXBuilderParameters; 027import java.security.cert.X509CertSelector; 028import java.time.Duration; 029import java.util.Arrays; 030import java.util.Objects; 031import java.util.concurrent.atomic.AtomicReference; 032import javax.net.ssl.CertPathTrustManagerParameters; 033import javax.net.ssl.KeyManager; 034import javax.net.ssl.KeyManagerFactory; 035import javax.net.ssl.TrustManager; 036import javax.net.ssl.TrustManagerFactory; 037import javax.net.ssl.X509ExtendedTrustManager; 038import javax.net.ssl.X509KeyManager; 039import javax.net.ssl.X509TrustManager; 040import org.apache.hadoop.conf.Configuration; 041import org.apache.hadoop.hbase.exceptions.KeyManagerException; 042import org.apache.hadoop.hbase.exceptions.SSLContextException; 043import org.apache.hadoop.hbase.exceptions.TrustManagerException; 044import org.apache.hadoop.hbase.exceptions.X509Exception; 045import org.apache.hadoop.hbase.io.FileChangeWatcher; 046import org.apache.yetus.audience.InterfaceAudience; 047import org.slf4j.Logger; 048import org.slf4j.LoggerFactory; 049 050import org.apache.hbase.thirdparty.io.netty.handler.ssl.OpenSsl; 051import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslContext; 052import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslContextBuilder; 053import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslProvider; 054 055/** 056 * Utility code for X509 handling Default cipher suites: Performance testing done by Facebook 057 * engineers shows that on Intel x86_64 machines, Java9 performs better with GCM and Java8 performs 058 * better with CBC, so these seem like reasonable defaults. 059 * <p/> 060 * This file has been copied from the Apache ZooKeeper project. 061 * @see <a href= 062 * "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/X509Util.java">Base 063 * revision</a> 064 */ 065@InterfaceAudience.Private 066public final class X509Util { 067 068 private static final Logger LOG = LoggerFactory.getLogger(X509Util.class); 069 private static final char[] EMPTY_CHAR_ARRAY = new char[0]; 070 071 // 072 // Common tls configs across both server and client 073 // 074 static final String CONFIG_PREFIX = "hbase.rpc.tls."; 075 public static final String TLS_CONFIG_PROTOCOL = CONFIG_PREFIX + "protocol"; 076 public static final String TLS_CONFIG_KEYSTORE_LOCATION = CONFIG_PREFIX + "keystore.location"; 077 public static final String TLS_CONFIG_KEYSTORE_TYPE = CONFIG_PREFIX + "keystore.type"; 078 public static final String TLS_CONFIG_KEYSTORE_PASSWORD = CONFIG_PREFIX + "keystore.password"; 079 public static final String TLS_CONFIG_TRUSTSTORE_LOCATION = CONFIG_PREFIX + "truststore.location"; 080 public static final String TLS_CONFIG_TRUSTSTORE_TYPE = CONFIG_PREFIX + "truststore.type"; 081 public static final String TLS_CONFIG_TRUSTSTORE_PASSWORD = CONFIG_PREFIX + "truststore.password"; 082 public static final String TLS_CONFIG_CLR = CONFIG_PREFIX + "clr"; 083 public static final String TLS_CONFIG_OCSP = CONFIG_PREFIX + "ocsp"; 084 public static final String TLS_CONFIG_REVERSE_DNS_LOOKUP_ENABLED = 085 CONFIG_PREFIX + "host-verification.reverse-dns.enabled"; 086 public static final String TLS_ENABLED_PROTOCOLS = CONFIG_PREFIX + "enabledProtocols"; 087 public static final String TLS_CIPHER_SUITES = CONFIG_PREFIX + "ciphersuites"; 088 public static final String TLS_CERT_RELOAD = CONFIG_PREFIX + "certReload"; 089 public static final String TLS_USE_OPENSSL = CONFIG_PREFIX + "useOpenSsl"; 090 091 // 092 // Server-side specific configs 093 // 094 public static final String HBASE_SERVER_NETTY_TLS_ENABLED = "hbase.server.netty.tls.enabled"; 095 public static final String HBASE_SERVER_NETTY_TLS_CLIENT_AUTH_MODE = 096 "hbase.server.netty.tls.client.auth.mode"; 097 public static final String HBASE_SERVER_NETTY_TLS_VERIFY_CLIENT_HOSTNAME = 098 "hbase.server.netty.tls.verify.client.hostname"; 099 public static final String HBASE_SERVER_NETTY_TLS_SUPPORTPLAINTEXT = 100 "hbase.server.netty.tls.supportplaintext"; 101 102 /** 103 * Set the SSL wrapSize for netty. This is only a maximum wrap size. Buffers smaller than this 104 * will not be consolidated, but buffers larger than this will be split into multiple wrap 105 * buffers. The netty default of 16k is not great for hbase which tends to return larger payloads 106 * than that, meaning most responses end up getting chunked up. This leads to more memory 107 * contention in netty's PoolArena. See https://github.com/netty/netty/pull/13551 108 */ 109 public static final String HBASE_SERVER_NETTY_TLS_WRAP_SIZE = "hbase.server.netty.tls.wrapSize"; 110 public static final int DEFAULT_HBASE_SERVER_NETTY_TLS_WRAP_SIZE = 1024 * 1024; 111 // 112 // Client-side specific configs 113 // 114 public static final String HBASE_CLIENT_NETTY_TLS_ENABLED = "hbase.client.netty.tls.enabled"; 115 public static final String HBASE_CLIENT_NETTY_TLS_VERIFY_SERVER_HOSTNAME = 116 "hbase.client.netty.tls.verify.server.hostname"; 117 public static final String HBASE_CLIENT_NETTY_TLS_HANDSHAKETIMEOUT = 118 "hbase.client.netty.tls.handshaketimeout"; 119 public static final int DEFAULT_HANDSHAKE_DETECTION_TIMEOUT_MILLIS = 5000; 120 121 public static final String HBASE_TLS_FILEPOLL_INTERVAL_MILLIS = 122 CONFIG_PREFIX + "filepoll.interval.millis"; 123 // 1 minute 124 private static final long DEFAULT_FILE_POLL_INTERVAL = Duration.ofSeconds(60).toMillis(); 125 126 /** 127 * Enum specifying the client auth requirement of server-side TLS sockets created by this 128 * X509Util. 129 * <ul> 130 * <li>NONE - do not request a client certificate.</li> 131 * <li>WANT - request a client certificate, but allow anonymous clients to connect.</li> 132 * <li>NEED - require a client certificate, disconnect anonymous clients.</li> 133 * </ul> 134 * If the config property is not set, the default value is NEED. 135 */ 136 public enum ClientAuth { 137 NONE(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth.NONE), 138 WANT(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth.OPTIONAL), 139 NEED(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth.REQUIRE); 140 141 private final org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth nettyAuth; 142 143 ClientAuth(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth nettyAuth) { 144 this.nettyAuth = nettyAuth; 145 } 146 147 /** 148 * Converts a property value to a ClientAuth enum. If the input string is empty or null, returns 149 * <code>ClientAuth.NEED</code>. 150 * @param prop the property string. 151 * @return the ClientAuth. 152 * @throws IllegalArgumentException if the property value is not "NONE", "WANT", "NEED", or 153 * empty/null. 154 */ 155 public static ClientAuth fromPropertyValue(String prop) { 156 if (prop == null || prop.length() == 0) { 157 return NEED; 158 } 159 return ClientAuth.valueOf(prop.toUpperCase()); 160 } 161 162 public org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth toNettyClientAuth() { 163 return nettyAuth; 164 } 165 } 166 167 private X509Util() { 168 // disabled 169 } 170 171 public static SslContext createSslContextForClient(Configuration config) 172 throws X509Exception, IOException { 173 174 SslContextBuilder sslContextBuilder = SslContextBuilder.forClient(); 175 176 configureOpenSslIfAvailable(sslContextBuilder, config); 177 String keyStoreLocation = config.get(TLS_CONFIG_KEYSTORE_LOCATION, ""); 178 char[] keyStorePassword = config.getPassword(TLS_CONFIG_KEYSTORE_PASSWORD); 179 String keyStoreType = config.get(TLS_CONFIG_KEYSTORE_TYPE, ""); 180 181 if (keyStoreLocation.isEmpty()) { 182 LOG.warn(TLS_CONFIG_KEYSTORE_LOCATION + " not specified"); 183 } else { 184 sslContextBuilder 185 .keyManager(createKeyManager(keyStoreLocation, keyStorePassword, keyStoreType)); 186 } 187 188 String trustStoreLocation = config.get(TLS_CONFIG_TRUSTSTORE_LOCATION, ""); 189 char[] trustStorePassword = config.getPassword(TLS_CONFIG_TRUSTSTORE_PASSWORD); 190 String trustStoreType = config.get(TLS_CONFIG_TRUSTSTORE_TYPE, ""); 191 192 boolean sslCrlEnabled = config.getBoolean(TLS_CONFIG_CLR, false); 193 boolean sslOcspEnabled = config.getBoolean(TLS_CONFIG_OCSP, false); 194 195 boolean verifyServerHostname = 196 config.getBoolean(HBASE_CLIENT_NETTY_TLS_VERIFY_SERVER_HOSTNAME, true); 197 boolean allowReverseDnsLookup = config.getBoolean(TLS_CONFIG_REVERSE_DNS_LOOKUP_ENABLED, true); 198 199 if (trustStoreLocation.isEmpty()) { 200 LOG.warn(TLS_CONFIG_TRUSTSTORE_LOCATION + " not specified"); 201 } else { 202 sslContextBuilder 203 .trustManager(createTrustManager(trustStoreLocation, trustStorePassword, trustStoreType, 204 sslCrlEnabled, sslOcspEnabled, verifyServerHostname, allowReverseDnsLookup)); 205 } 206 207 sslContextBuilder.enableOcsp(sslOcspEnabled); 208 String[] enabledProtocols = getEnabledProtocols(config); 209 if (enabledProtocols != null) { 210 sslContextBuilder.protocols(enabledProtocols); 211 } 212 String[] cipherSuites = getCipherSuites(config); 213 if (cipherSuites != null) { 214 sslContextBuilder.ciphers(Arrays.asList(cipherSuites)); 215 } 216 217 return sslContextBuilder.build(); 218 } 219 220 /** 221 * Adds SslProvider.OPENSSL if OpenSsl is available and enabled. In order to make it available, 222 * one must ensure that a properly shaded netty-tcnative is on the classpath. Properly shaded 223 * means relocated to be prefixed with "org.apache.hbase.thirdparty" like the rest of the netty 224 * classes. We make available org.apache.hbase:hbase-openssl as a convenience module which one can 225 * use to pull in a shaded netty-tcnative statically linked against boringssl. 226 */ 227 private static boolean configureOpenSslIfAvailable(SslContextBuilder sslContextBuilder, 228 Configuration conf) { 229 if (OpenSsl.isAvailable() && conf.getBoolean(TLS_USE_OPENSSL, true)) { 230 LOG.debug("Using netty-tcnative to accelerate SSL handling"); 231 sslContextBuilder.sslProvider(SslProvider.OPENSSL); 232 return true; 233 } else { 234 if (LOG.isDebugEnabled()) { 235 LOG.debug("Using default JDK SSL provider because netty-tcnative is not {}", 236 OpenSsl.isAvailable() ? "enabled" : "available"); 237 } 238 sslContextBuilder.sslProvider(SslProvider.JDK); 239 return false; 240 } 241 } 242 243 public static SslContext createSslContextForServer(Configuration config) 244 throws X509Exception, IOException { 245 String keyStoreLocation = config.get(TLS_CONFIG_KEYSTORE_LOCATION, ""); 246 char[] keyStorePassword = config.getPassword(TLS_CONFIG_KEYSTORE_PASSWORD); 247 String keyStoreType = config.get(TLS_CONFIG_KEYSTORE_TYPE, ""); 248 249 if (keyStoreLocation.isEmpty()) { 250 throw new SSLContextException( 251 "Keystore is required for SSL server: " + TLS_CONFIG_KEYSTORE_LOCATION); 252 } 253 254 SslContextBuilder sslContextBuilder; 255 sslContextBuilder = SslContextBuilder 256 .forServer(createKeyManager(keyStoreLocation, keyStorePassword, keyStoreType)); 257 258 configureOpenSslIfAvailable(sslContextBuilder, config); 259 String trustStoreLocation = config.get(TLS_CONFIG_TRUSTSTORE_LOCATION, ""); 260 char[] trustStorePassword = config.getPassword(TLS_CONFIG_TRUSTSTORE_PASSWORD); 261 String trustStoreType = config.get(TLS_CONFIG_TRUSTSTORE_TYPE, ""); 262 263 boolean sslCrlEnabled = config.getBoolean(TLS_CONFIG_CLR, false); 264 boolean sslOcspEnabled = config.getBoolean(TLS_CONFIG_OCSP, false); 265 266 ClientAuth clientAuth = 267 ClientAuth.fromPropertyValue(config.get(HBASE_SERVER_NETTY_TLS_CLIENT_AUTH_MODE)); 268 boolean verifyClientHostname = 269 config.getBoolean(HBASE_SERVER_NETTY_TLS_VERIFY_CLIENT_HOSTNAME, true); 270 boolean allowReverseDnsLookup = config.getBoolean(TLS_CONFIG_REVERSE_DNS_LOOKUP_ENABLED, true); 271 272 if (trustStoreLocation.isEmpty()) { 273 LOG.warn(TLS_CONFIG_TRUSTSTORE_LOCATION + " not specified"); 274 } else { 275 sslContextBuilder 276 .trustManager(createTrustManager(trustStoreLocation, trustStorePassword, trustStoreType, 277 sslCrlEnabled, sslOcspEnabled, verifyClientHostname, allowReverseDnsLookup)); 278 } 279 280 sslContextBuilder.enableOcsp(sslOcspEnabled); 281 String[] enabledProtocols = getEnabledProtocols(config); 282 if (enabledProtocols != null) { 283 sslContextBuilder.protocols(enabledProtocols); 284 } 285 String[] cipherSuites = getCipherSuites(config); 286 if (cipherSuites != null) { 287 sslContextBuilder.ciphers(Arrays.asList(cipherSuites)); 288 } 289 sslContextBuilder.clientAuth(clientAuth.toNettyClientAuth()); 290 291 return sslContextBuilder.build(); 292 } 293 294 /** 295 * Creates a key manager by loading the key store from the given file of the given type, 296 * optionally decrypting it using the given password. 297 * @param keyStoreLocation the location of the key store file. 298 * @param keyStorePassword optional password to decrypt the key store. If empty, assumes the key 299 * store is not encrypted. 300 * @param keyStoreType must be JKS, PEM, PKCS12, BCFKS or null. If null, attempts to 301 * autodetect the key store type from the file extension (e.g. .jks / 302 * .pem). 303 * @return the key manager. 304 * @throws KeyManagerException if something goes wrong. 305 */ 306 static X509KeyManager createKeyManager(String keyStoreLocation, char[] keyStorePassword, 307 String keyStoreType) throws KeyManagerException { 308 309 if (keyStorePassword == null) { 310 keyStorePassword = EMPTY_CHAR_ARRAY; 311 } 312 313 try { 314 KeyStoreFileType storeFileType = 315 KeyStoreFileType.fromPropertyValueOrFileName(keyStoreType, keyStoreLocation); 316 KeyStore ks = FileKeyStoreLoaderBuilderProvider.getBuilderForKeyStoreFileType(storeFileType) 317 .setKeyStorePath(keyStoreLocation).setKeyStorePassword(keyStorePassword).build() 318 .loadKeyStore(); 319 320 KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX"); 321 kmf.init(ks, keyStorePassword); 322 323 for (KeyManager km : kmf.getKeyManagers()) { 324 if (km instanceof X509KeyManager) { 325 return (X509KeyManager) km; 326 } 327 } 328 throw new KeyManagerException("Couldn't find X509KeyManager"); 329 } catch (IOException | GeneralSecurityException | IllegalArgumentException e) { 330 throw new KeyManagerException(e); 331 } 332 } 333 334 /** 335 * Creates a trust manager by loading the trust store from the given file of the given type, 336 * optionally decrypting it using the given password. 337 * @param trustStoreLocation the location of the trust store file. 338 * @param trustStorePassword optional password to decrypt the trust store (only applies to JKS 339 * trust stores). If empty, assumes the trust store is not encrypted. 340 * @param trustStoreType must be JKS, PEM, PKCS12, BCFKS or null. If null, attempts to 341 * autodetect the trust store type from the file extension (e.g. .jks 342 * / .pem). 343 * @param crlEnabled enable CRL (certificate revocation list) checks. 344 * @param ocspEnabled enable OCSP (online certificate status protocol) checks. 345 * @param verifyHostName if true, ssl peer hostname must match name in certificate 346 * @param allowReverseDnsLookup if true, allow falling back to reverse dns lookup in verifying 347 * hostname 348 * @return the trust manager. 349 * @throws TrustManagerException if something goes wrong. 350 */ 351 static X509TrustManager createTrustManager(String trustStoreLocation, char[] trustStorePassword, 352 String trustStoreType, boolean crlEnabled, boolean ocspEnabled, boolean verifyHostName, 353 boolean allowReverseDnsLookup) throws TrustManagerException { 354 355 if (trustStorePassword == null) { 356 trustStorePassword = EMPTY_CHAR_ARRAY; 357 } 358 359 try { 360 KeyStoreFileType storeFileType = 361 KeyStoreFileType.fromPropertyValueOrFileName(trustStoreType, trustStoreLocation); 362 KeyStore ts = FileKeyStoreLoaderBuilderProvider.getBuilderForKeyStoreFileType(storeFileType) 363 .setTrustStorePath(trustStoreLocation).setTrustStorePassword(trustStorePassword).build() 364 .loadTrustStore(); 365 366 PKIXBuilderParameters pbParams = new PKIXBuilderParameters(ts, new X509CertSelector()); 367 if (crlEnabled || ocspEnabled) { 368 pbParams.setRevocationEnabled(true); 369 System.setProperty("com.sun.net.ssl.checkRevocation", "true"); 370 if (crlEnabled) { 371 System.setProperty("com.sun.security.enableCRLDP", "true"); 372 } 373 if (ocspEnabled) { 374 Security.setProperty("ocsp.enable", "true"); 375 } 376 } else { 377 pbParams.setRevocationEnabled(false); 378 } 379 380 // Revocation checking is only supported with the PKIX algorithm 381 TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX"); 382 tmf.init(new CertPathTrustManagerParameters(pbParams)); 383 384 for (final TrustManager tm : tmf.getTrustManagers()) { 385 if (tm instanceof X509ExtendedTrustManager) { 386 return new HBaseTrustManager((X509ExtendedTrustManager) tm, verifyHostName, 387 allowReverseDnsLookup); 388 } 389 } 390 throw new TrustManagerException("Couldn't find X509TrustManager"); 391 } catch (IOException | GeneralSecurityException | IllegalArgumentException e) { 392 throw new TrustManagerException(e); 393 } 394 } 395 396 private static String[] getEnabledProtocols(Configuration config) { 397 String enabledProtocolsInput = config.get(TLS_ENABLED_PROTOCOLS); 398 if (enabledProtocolsInput == null) { 399 enabledProtocolsInput = config.get(TLS_CONFIG_PROTOCOL); 400 } 401 if (enabledProtocolsInput != null) { 402 return enabledProtocolsInput.split(","); 403 } else { 404 return null; 405 } 406 } 407 408 private static String[] getCipherSuites(Configuration config) { 409 String cipherSuitesInput = config.get(TLS_CIPHER_SUITES); 410 if (cipherSuitesInput == null) { 411 return null; 412 } else { 413 return cipherSuitesInput.split(","); 414 } 415 } 416 417 /** 418 * Enable certificate file reloading by creating FileWatchers for keystore and truststore. 419 * AtomicReferences will be set with the new instances. resetContext - if not null - will be 420 * called when the file has been modified. 421 * @param keystoreWatcher Reference to keystoreFileWatcher. 422 * @param trustStoreWatcher Reference to truststoreFileWatcher. 423 * @param resetContext Callback for file changes. 424 */ 425 public static void enableCertFileReloading(Configuration config, 426 AtomicReference<FileChangeWatcher> keystoreWatcher, 427 AtomicReference<FileChangeWatcher> trustStoreWatcher, Runnable resetContext) 428 throws IOException { 429 String keyStoreLocation = config.get(TLS_CONFIG_KEYSTORE_LOCATION, ""); 430 keystoreWatcher.set(newFileChangeWatcher(config, keyStoreLocation, resetContext)); 431 String trustStoreLocation = config.get(TLS_CONFIG_TRUSTSTORE_LOCATION, ""); 432 // we are using the same callback for both. there's no reason to kick off two 433 // threads if keystore/truststore are both at the same location 434 if (!keyStoreLocation.equals(trustStoreLocation)) { 435 trustStoreWatcher.set(newFileChangeWatcher(config, trustStoreLocation, resetContext)); 436 } 437 } 438 439 private static FileChangeWatcher newFileChangeWatcher(Configuration config, String fileLocation, 440 Runnable resetContext) throws IOException { 441 if (fileLocation == null || fileLocation.isEmpty() || resetContext == null) { 442 return null; 443 } 444 final Path filePath = Paths.get(fileLocation).toAbsolutePath(); 445 FileChangeWatcher fileChangeWatcher = 446 new FileChangeWatcher(filePath, Objects.toString(filePath.getFileName()), 447 Duration 448 .ofMillis(config.getLong(HBASE_TLS_FILEPOLL_INTERVAL_MILLIS, DEFAULT_FILE_POLL_INTERVAL)), 449 watchEventFilePath -> handleWatchEvent(watchEventFilePath, resetContext)); 450 fileChangeWatcher.start(); 451 return fileChangeWatcher; 452 } 453 454 /** 455 * Handler for watch events that let us know a file we may care about has changed on disk. 456 */ 457 private static void handleWatchEvent(Path filePath, Runnable resetContext) { 458 LOG.info("Attempting to reset default SSL context after receiving watch event on file {}", 459 filePath); 460 resetContext.run(); 461 } 462}