001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.io.crypto.tls; 019 020import java.io.IOException; 021import java.nio.file.Path; 022import java.nio.file.Paths; 023import java.security.GeneralSecurityException; 024import java.security.KeyStore; 025import java.security.Security; 026import java.security.cert.PKIXBuilderParameters; 027import java.security.cert.X509CertSelector; 028import java.time.Duration; 029import java.util.Arrays; 030import java.util.Objects; 031import java.util.concurrent.atomic.AtomicReference; 032import javax.net.ssl.CertPathTrustManagerParameters; 033import javax.net.ssl.KeyManager; 034import javax.net.ssl.KeyManagerFactory; 035import javax.net.ssl.TrustManager; 036import javax.net.ssl.TrustManagerFactory; 037import javax.net.ssl.X509ExtendedTrustManager; 038import javax.net.ssl.X509KeyManager; 039import javax.net.ssl.X509TrustManager; 040import org.apache.hadoop.conf.Configuration; 041import org.apache.hadoop.hbase.exceptions.KeyManagerException; 042import org.apache.hadoop.hbase.exceptions.SSLContextException; 043import org.apache.hadoop.hbase.exceptions.TrustManagerException; 044import org.apache.hadoop.hbase.exceptions.X509Exception; 045import org.apache.hadoop.hbase.io.FileChangeWatcher; 046import org.apache.yetus.audience.InterfaceAudience; 047import org.slf4j.Logger; 048import org.slf4j.LoggerFactory; 049 050import org.apache.hbase.thirdparty.io.netty.handler.ssl.OpenSsl; 051import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslContext; 052import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslContextBuilder; 053import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslProvider; 054 055/** 056 * Utility code for X509 handling Default cipher suites: Performance testing done by Facebook 057 * engineers shows that on Intel x86_64 machines, Java9 performs better with GCM and Java8 performs 058 * better with CBC, so these seem like reasonable defaults. 059 * <p/> 060 * This file has been copied from the Apache ZooKeeper project. 061 * @see <a href= 062 * "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/X509Util.java">Base 063 * revision</a> 064 */ 065@InterfaceAudience.Private 066public final class X509Util { 067 068 private static final Logger LOG = LoggerFactory.getLogger(X509Util.class); 069 private static final char[] EMPTY_CHAR_ARRAY = new char[0]; 070 071 // 072 // Common tls configs across both server and client 073 // 074 static final String CONFIG_PREFIX = "hbase.rpc.tls."; 075 public static final String TLS_CONFIG_PROTOCOL = CONFIG_PREFIX + "protocol"; 076 public static final String TLS_CONFIG_KEYSTORE_LOCATION = CONFIG_PREFIX + "keystore.location"; 077 public static final String TLS_CONFIG_KEYSTORE_TYPE = CONFIG_PREFIX + "keystore.type"; 078 public static final String TLS_CONFIG_KEYSTORE_PASSWORD = CONFIG_PREFIX + "keystore.password"; 079 public static final String TLS_CONFIG_TRUSTSTORE_LOCATION = CONFIG_PREFIX + "truststore.location"; 080 public static final String TLS_CONFIG_TRUSTSTORE_TYPE = CONFIG_PREFIX + "truststore.type"; 081 public static final String TLS_CONFIG_TRUSTSTORE_PASSWORD = CONFIG_PREFIX + "truststore.password"; 082 public static final String TLS_CONFIG_CLR = CONFIG_PREFIX + "clr"; 083 public static final String TLS_CONFIG_OCSP = CONFIG_PREFIX + "ocsp"; 084 public static final String TLS_CONFIG_REVERSE_DNS_LOOKUP_ENABLED = 085 CONFIG_PREFIX + "host-verification.reverse-dns.enabled"; 086 public static final String TLS_ENABLED_PROTOCOLS = CONFIG_PREFIX + "enabledProtocols"; 087 public static final String TLS_CIPHER_SUITES = CONFIG_PREFIX + "ciphersuites"; 088 public static final String TLS_CERT_RELOAD = CONFIG_PREFIX + "certReload"; 089 public static final String TLS_USE_OPENSSL = CONFIG_PREFIX + "useOpenSsl"; 090 public static final String DEFAULT_PROTOCOL = "TLSv1.2"; 091 092 // 093 // Server-side specific configs 094 // 095 public static final String HBASE_SERVER_NETTY_TLS_ENABLED = "hbase.server.netty.tls.enabled"; 096 public static final String HBASE_SERVER_NETTY_TLS_CLIENT_AUTH_MODE = 097 "hbase.server.netty.tls.client.auth.mode"; 098 public static final String HBASE_SERVER_NETTY_TLS_VERIFY_CLIENT_HOSTNAME = 099 "hbase.server.netty.tls.verify.client.hostname"; 100 public static final String HBASE_SERVER_NETTY_TLS_SUPPORTPLAINTEXT = 101 "hbase.server.netty.tls.supportplaintext"; 102 103 /** 104 * Set the SSL wrapSize for netty. This is only a maximum wrap size. Buffers smaller than this 105 * will not be consolidated, but buffers larger than this will be split into multiple wrap 106 * buffers. The netty default of 16k is not great for hbase which tends to return larger payloads 107 * than that, meaning most responses end up getting chunked up. This leads to more memory 108 * contention in netty's PoolArena. See https://github.com/netty/netty/pull/13551 109 */ 110 public static final String HBASE_SERVER_NETTY_TLS_WRAP_SIZE = "hbase.server.netty.tls.wrapSize"; 111 public static final int DEFAULT_HBASE_SERVER_NETTY_TLS_WRAP_SIZE = 1024 * 1024; 112 // 113 // Client-side specific configs 114 // 115 public static final String HBASE_CLIENT_NETTY_TLS_ENABLED = "hbase.client.netty.tls.enabled"; 116 public static final String HBASE_CLIENT_NETTY_TLS_VERIFY_SERVER_HOSTNAME = 117 "hbase.client.netty.tls.verify.server.hostname"; 118 public static final String HBASE_CLIENT_NETTY_TLS_HANDSHAKETIMEOUT = 119 "hbase.client.netty.tls.handshaketimeout"; 120 public static final int DEFAULT_HANDSHAKE_DETECTION_TIMEOUT_MILLIS = 5000; 121 122 public static final String HBASE_TLS_FILEPOLL_INTERVAL_MILLIS = 123 CONFIG_PREFIX + "filepoll.interval.millis"; 124 // 1 minute 125 private static final long DEFAULT_FILE_POLL_INTERVAL = Duration.ofSeconds(60).toMillis(); 126 127 /** 128 * Enum specifying the client auth requirement of server-side TLS sockets created by this 129 * X509Util. 130 * <ul> 131 * <li>NONE - do not request a client certificate.</li> 132 * <li>WANT - request a client certificate, but allow anonymous clients to connect.</li> 133 * <li>NEED - require a client certificate, disconnect anonymous clients.</li> 134 * </ul> 135 * If the config property is not set, the default value is NEED. 136 */ 137 public enum ClientAuth { 138 NONE(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth.NONE), 139 WANT(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth.OPTIONAL), 140 NEED(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth.REQUIRE); 141 142 private final org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth nettyAuth; 143 144 ClientAuth(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth nettyAuth) { 145 this.nettyAuth = nettyAuth; 146 } 147 148 /** 149 * Converts a property value to a ClientAuth enum. If the input string is empty or null, returns 150 * <code>ClientAuth.NEED</code>. 151 * @param prop the property string. 152 * @return the ClientAuth. 153 * @throws IllegalArgumentException if the property value is not "NONE", "WANT", "NEED", or 154 * empty/null. 155 */ 156 public static ClientAuth fromPropertyValue(String prop) { 157 if (prop == null || prop.length() == 0) { 158 return NEED; 159 } 160 return ClientAuth.valueOf(prop.toUpperCase()); 161 } 162 163 public org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth toNettyClientAuth() { 164 return nettyAuth; 165 } 166 } 167 168 private X509Util() { 169 // disabled 170 } 171 172 public static SslContext createSslContextForClient(Configuration config) 173 throws X509Exception, IOException { 174 175 SslContextBuilder sslContextBuilder = SslContextBuilder.forClient(); 176 177 configureOpenSslIfAvailable(sslContextBuilder, config); 178 String keyStoreLocation = config.get(TLS_CONFIG_KEYSTORE_LOCATION, ""); 179 char[] keyStorePassword = config.getPassword(TLS_CONFIG_KEYSTORE_PASSWORD); 180 String keyStoreType = config.get(TLS_CONFIG_KEYSTORE_TYPE, ""); 181 182 if (keyStoreLocation.isEmpty()) { 183 LOG.warn(TLS_CONFIG_KEYSTORE_LOCATION + " not specified"); 184 } else { 185 sslContextBuilder 186 .keyManager(createKeyManager(keyStoreLocation, keyStorePassword, keyStoreType)); 187 } 188 189 String trustStoreLocation = config.get(TLS_CONFIG_TRUSTSTORE_LOCATION, ""); 190 char[] trustStorePassword = config.getPassword(TLS_CONFIG_TRUSTSTORE_PASSWORD); 191 String trustStoreType = config.get(TLS_CONFIG_TRUSTSTORE_TYPE, ""); 192 193 boolean sslCrlEnabled = config.getBoolean(TLS_CONFIG_CLR, false); 194 boolean sslOcspEnabled = config.getBoolean(TLS_CONFIG_OCSP, false); 195 196 boolean verifyServerHostname = 197 config.getBoolean(HBASE_CLIENT_NETTY_TLS_VERIFY_SERVER_HOSTNAME, true); 198 boolean allowReverseDnsLookup = config.getBoolean(TLS_CONFIG_REVERSE_DNS_LOOKUP_ENABLED, true); 199 200 if (trustStoreLocation.isEmpty()) { 201 LOG.warn(TLS_CONFIG_TRUSTSTORE_LOCATION + " not specified"); 202 } else { 203 sslContextBuilder 204 .trustManager(createTrustManager(trustStoreLocation, trustStorePassword, trustStoreType, 205 sslCrlEnabled, sslOcspEnabled, verifyServerHostname, allowReverseDnsLookup)); 206 } 207 208 sslContextBuilder.enableOcsp(sslOcspEnabled); 209 sslContextBuilder.protocols(getEnabledProtocols(config)); 210 String[] cipherSuites = getCipherSuites(config); 211 if (cipherSuites != null) { 212 sslContextBuilder.ciphers(Arrays.asList(cipherSuites)); 213 } 214 215 return sslContextBuilder.build(); 216 } 217 218 /** 219 * Adds SslProvider.OPENSSL if OpenSsl is available and enabled. In order to make it available, 220 * one must ensure that a properly shaded netty-tcnative is on the classpath. Properly shaded 221 * means relocated to be prefixed with "org.apache.hbase.thirdparty" like the rest of the netty 222 * classes. We make available org.apache.hbase:hbase-openssl as a convenience module which one can 223 * use to pull in a shaded netty-tcnative statically linked against boringssl. 224 */ 225 private static boolean configureOpenSslIfAvailable(SslContextBuilder sslContextBuilder, 226 Configuration conf) { 227 if (OpenSsl.isAvailable() && conf.getBoolean(TLS_USE_OPENSSL, true)) { 228 LOG.debug("Using netty-tcnative to accelerate SSL handling"); 229 sslContextBuilder.sslProvider(SslProvider.OPENSSL); 230 return true; 231 } else { 232 if (LOG.isDebugEnabled()) { 233 LOG.debug("Using default JDK SSL provider because netty-tcnative is not {}", 234 OpenSsl.isAvailable() ? "enabled" : "available"); 235 } 236 sslContextBuilder.sslProvider(SslProvider.JDK); 237 return false; 238 } 239 } 240 241 public static SslContext createSslContextForServer(Configuration config) 242 throws X509Exception, IOException { 243 String keyStoreLocation = config.get(TLS_CONFIG_KEYSTORE_LOCATION, ""); 244 char[] keyStorePassword = config.getPassword(TLS_CONFIG_KEYSTORE_PASSWORD); 245 String keyStoreType = config.get(TLS_CONFIG_KEYSTORE_TYPE, ""); 246 247 if (keyStoreLocation.isEmpty()) { 248 throw new SSLContextException( 249 "Keystore is required for SSL server: " + TLS_CONFIG_KEYSTORE_LOCATION); 250 } 251 252 SslContextBuilder sslContextBuilder; 253 sslContextBuilder = SslContextBuilder 254 .forServer(createKeyManager(keyStoreLocation, keyStorePassword, keyStoreType)); 255 256 configureOpenSslIfAvailable(sslContextBuilder, config); 257 String trustStoreLocation = config.get(TLS_CONFIG_TRUSTSTORE_LOCATION, ""); 258 char[] trustStorePassword = config.getPassword(TLS_CONFIG_TRUSTSTORE_PASSWORD); 259 String trustStoreType = config.get(TLS_CONFIG_TRUSTSTORE_TYPE, ""); 260 261 boolean sslCrlEnabled = config.getBoolean(TLS_CONFIG_CLR, false); 262 boolean sslOcspEnabled = config.getBoolean(TLS_CONFIG_OCSP, false); 263 264 ClientAuth clientAuth = 265 ClientAuth.fromPropertyValue(config.get(HBASE_SERVER_NETTY_TLS_CLIENT_AUTH_MODE)); 266 boolean verifyClientHostname = 267 config.getBoolean(HBASE_SERVER_NETTY_TLS_VERIFY_CLIENT_HOSTNAME, true); 268 boolean allowReverseDnsLookup = config.getBoolean(TLS_CONFIG_REVERSE_DNS_LOOKUP_ENABLED, true); 269 270 if (trustStoreLocation.isEmpty()) { 271 LOG.warn(TLS_CONFIG_TRUSTSTORE_LOCATION + " not specified"); 272 } else { 273 sslContextBuilder 274 .trustManager(createTrustManager(trustStoreLocation, trustStorePassword, trustStoreType, 275 sslCrlEnabled, sslOcspEnabled, verifyClientHostname, allowReverseDnsLookup)); 276 } 277 278 sslContextBuilder.enableOcsp(sslOcspEnabled); 279 sslContextBuilder.protocols(getEnabledProtocols(config)); 280 String[] cipherSuites = getCipherSuites(config); 281 if (cipherSuites != null) { 282 sslContextBuilder.ciphers(Arrays.asList(cipherSuites)); 283 } 284 sslContextBuilder.clientAuth(clientAuth.toNettyClientAuth()); 285 286 return sslContextBuilder.build(); 287 } 288 289 /** 290 * Creates a key manager by loading the key store from the given file of the given type, 291 * optionally decrypting it using the given password. 292 * @param keyStoreLocation the location of the key store file. 293 * @param keyStorePassword optional password to decrypt the key store. If empty, assumes the key 294 * store is not encrypted. 295 * @param keyStoreType must be JKS, PEM, PKCS12, BCFKS or null. If null, attempts to 296 * autodetect the key store type from the file extension (e.g. .jks / 297 * .pem). 298 * @return the key manager. 299 * @throws KeyManagerException if something goes wrong. 300 */ 301 static X509KeyManager createKeyManager(String keyStoreLocation, char[] keyStorePassword, 302 String keyStoreType) throws KeyManagerException { 303 304 if (keyStorePassword == null) { 305 keyStorePassword = EMPTY_CHAR_ARRAY; 306 } 307 308 try { 309 KeyStoreFileType storeFileType = 310 KeyStoreFileType.fromPropertyValueOrFileName(keyStoreType, keyStoreLocation); 311 KeyStore ks = FileKeyStoreLoaderBuilderProvider.getBuilderForKeyStoreFileType(storeFileType) 312 .setKeyStorePath(keyStoreLocation).setKeyStorePassword(keyStorePassword).build() 313 .loadKeyStore(); 314 315 KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX"); 316 kmf.init(ks, keyStorePassword); 317 318 for (KeyManager km : kmf.getKeyManagers()) { 319 if (km instanceof X509KeyManager) { 320 return (X509KeyManager) km; 321 } 322 } 323 throw new KeyManagerException("Couldn't find X509KeyManager"); 324 } catch (IOException | GeneralSecurityException | IllegalArgumentException e) { 325 throw new KeyManagerException(e); 326 } 327 } 328 329 /** 330 * Creates a trust manager by loading the trust store from the given file of the given type, 331 * optionally decrypting it using the given password. 332 * @param trustStoreLocation the location of the trust store file. 333 * @param trustStorePassword optional password to decrypt the trust store (only applies to JKS 334 * trust stores). If empty, assumes the trust store is not encrypted. 335 * @param trustStoreType must be JKS, PEM, PKCS12, BCFKS or null. If null, attempts to 336 * autodetect the trust store type from the file extension (e.g. .jks 337 * / .pem). 338 * @param crlEnabled enable CRL (certificate revocation list) checks. 339 * @param ocspEnabled enable OCSP (online certificate status protocol) checks. 340 * @param verifyHostName if true, ssl peer hostname must match name in certificate 341 * @param allowReverseDnsLookup if true, allow falling back to reverse dns lookup in verifying 342 * hostname 343 * @return the trust manager. 344 * @throws TrustManagerException if something goes wrong. 345 */ 346 static X509TrustManager createTrustManager(String trustStoreLocation, char[] trustStorePassword, 347 String trustStoreType, boolean crlEnabled, boolean ocspEnabled, boolean verifyHostName, 348 boolean allowReverseDnsLookup) throws TrustManagerException { 349 350 if (trustStorePassword == null) { 351 trustStorePassword = EMPTY_CHAR_ARRAY; 352 } 353 354 try { 355 KeyStoreFileType storeFileType = 356 KeyStoreFileType.fromPropertyValueOrFileName(trustStoreType, trustStoreLocation); 357 KeyStore ts = FileKeyStoreLoaderBuilderProvider.getBuilderForKeyStoreFileType(storeFileType) 358 .setTrustStorePath(trustStoreLocation).setTrustStorePassword(trustStorePassword).build() 359 .loadTrustStore(); 360 361 PKIXBuilderParameters pbParams = new PKIXBuilderParameters(ts, new X509CertSelector()); 362 if (crlEnabled || ocspEnabled) { 363 pbParams.setRevocationEnabled(true); 364 System.setProperty("com.sun.net.ssl.checkRevocation", "true"); 365 if (crlEnabled) { 366 System.setProperty("com.sun.security.enableCRLDP", "true"); 367 } 368 if (ocspEnabled) { 369 Security.setProperty("ocsp.enable", "true"); 370 } 371 } else { 372 pbParams.setRevocationEnabled(false); 373 } 374 375 // Revocation checking is only supported with the PKIX algorithm 376 TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX"); 377 tmf.init(new CertPathTrustManagerParameters(pbParams)); 378 379 for (final TrustManager tm : tmf.getTrustManagers()) { 380 if (tm instanceof X509ExtendedTrustManager) { 381 return new HBaseTrustManager((X509ExtendedTrustManager) tm, verifyHostName, 382 allowReverseDnsLookup); 383 } 384 } 385 throw new TrustManagerException("Couldn't find X509TrustManager"); 386 } catch (IOException | GeneralSecurityException | IllegalArgumentException e) { 387 throw new TrustManagerException(e); 388 } 389 } 390 391 private static String[] getEnabledProtocols(Configuration config) { 392 String enabledProtocolsInput = config.get(TLS_ENABLED_PROTOCOLS); 393 if (enabledProtocolsInput == null) { 394 return new String[] { config.get(TLS_CONFIG_PROTOCOL, DEFAULT_PROTOCOL) }; 395 } 396 return enabledProtocolsInput.split(","); 397 } 398 399 private static String[] getCipherSuites(Configuration config) { 400 String cipherSuitesInput = config.get(TLS_CIPHER_SUITES); 401 if (cipherSuitesInput == null) { 402 return null; 403 } else { 404 return cipherSuitesInput.split(","); 405 } 406 } 407 408 /** 409 * Enable certificate file reloading by creating FileWatchers for keystore and truststore. 410 * AtomicReferences will be set with the new instances. resetContext - if not null - will be 411 * called when the file has been modified. 412 * @param keystoreWatcher Reference to keystoreFileWatcher. 413 * @param trustStoreWatcher Reference to truststoreFileWatcher. 414 * @param resetContext Callback for file changes. 415 */ 416 public static void enableCertFileReloading(Configuration config, 417 AtomicReference<FileChangeWatcher> keystoreWatcher, 418 AtomicReference<FileChangeWatcher> trustStoreWatcher, Runnable resetContext) 419 throws IOException { 420 String keyStoreLocation = config.get(TLS_CONFIG_KEYSTORE_LOCATION, ""); 421 keystoreWatcher.set(newFileChangeWatcher(config, keyStoreLocation, resetContext)); 422 String trustStoreLocation = config.get(TLS_CONFIG_TRUSTSTORE_LOCATION, ""); 423 // we are using the same callback for both. there's no reason to kick off two 424 // threads if keystore/truststore are both at the same location 425 if (!keyStoreLocation.equals(trustStoreLocation)) { 426 trustStoreWatcher.set(newFileChangeWatcher(config, trustStoreLocation, resetContext)); 427 } 428 } 429 430 private static FileChangeWatcher newFileChangeWatcher(Configuration config, String fileLocation, 431 Runnable resetContext) throws IOException { 432 if (fileLocation == null || fileLocation.isEmpty() || resetContext == null) { 433 return null; 434 } 435 final Path filePath = Paths.get(fileLocation).toAbsolutePath(); 436 FileChangeWatcher fileChangeWatcher = 437 new FileChangeWatcher(filePath, Objects.toString(filePath.getFileName()), 438 Duration 439 .ofMillis(config.getLong(HBASE_TLS_FILEPOLL_INTERVAL_MILLIS, DEFAULT_FILE_POLL_INTERVAL)), 440 watchEventFilePath -> handleWatchEvent(watchEventFilePath, resetContext)); 441 fileChangeWatcher.start(); 442 return fileChangeWatcher; 443 } 444 445 /** 446 * Handler for watch events that let us know a file we may care about has changed on disk. 447 */ 448 private static void handleWatchEvent(Path filePath, Runnable resetContext) { 449 LOG.info("Attempting to reset default SSL context after receiving watch event on file {}", 450 filePath); 451 resetContext.run(); 452 } 453}