001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.crypto.tls;
019
020import java.io.IOException;
021import java.nio.file.Path;
022import java.nio.file.Paths;
023import java.security.GeneralSecurityException;
024import java.security.KeyStore;
025import java.security.Security;
026import java.security.cert.PKIXBuilderParameters;
027import java.security.cert.X509CertSelector;
028import java.time.Duration;
029import java.util.Arrays;
030import java.util.Objects;
031import java.util.concurrent.atomic.AtomicReference;
032import javax.net.ssl.CertPathTrustManagerParameters;
033import javax.net.ssl.KeyManager;
034import javax.net.ssl.KeyManagerFactory;
035import javax.net.ssl.TrustManager;
036import javax.net.ssl.TrustManagerFactory;
037import javax.net.ssl.X509ExtendedTrustManager;
038import javax.net.ssl.X509KeyManager;
039import javax.net.ssl.X509TrustManager;
040import org.apache.hadoop.conf.Configuration;
041import org.apache.hadoop.hbase.exceptions.KeyManagerException;
042import org.apache.hadoop.hbase.exceptions.SSLContextException;
043import org.apache.hadoop.hbase.exceptions.TrustManagerException;
044import org.apache.hadoop.hbase.exceptions.X509Exception;
045import org.apache.hadoop.hbase.io.FileChangeWatcher;
046import org.apache.yetus.audience.InterfaceAudience;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050import org.apache.hbase.thirdparty.io.netty.handler.ssl.OpenSsl;
051import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslContext;
052import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslContextBuilder;
053import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslProvider;
054
055/**
056 * Utility code for X509 handling Default cipher suites: Performance testing done by Facebook
057 * engineers shows that on Intel x86_64 machines, Java9 performs better with GCM and Java8 performs
058 * better with CBC, so these seem like reasonable defaults.
059 * <p/>
060 * This file has been copied from the Apache ZooKeeper project.
061 * @see <a href=
062 *      "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/X509Util.java">Base
063 *      revision</a>
064 */
065@InterfaceAudience.Private
066public final class X509Util {
067
068  private static final Logger LOG = LoggerFactory.getLogger(X509Util.class);
069  private static final char[] EMPTY_CHAR_ARRAY = new char[0];
070
071  //
072  // Common tls configs across both server and client
073  //
074  static final String CONFIG_PREFIX = "hbase.rpc.tls.";
075  public static final String TLS_CONFIG_PROTOCOL = CONFIG_PREFIX + "protocol";
076  public static final String TLS_CONFIG_KEYSTORE_LOCATION = CONFIG_PREFIX + "keystore.location";
077  public static final String TLS_CONFIG_KEYSTORE_TYPE = CONFIG_PREFIX + "keystore.type";
078  public static final String TLS_CONFIG_KEYSTORE_PASSWORD = CONFIG_PREFIX + "keystore.password";
079  public static final String TLS_CONFIG_TRUSTSTORE_LOCATION = CONFIG_PREFIX + "truststore.location";
080  public static final String TLS_CONFIG_TRUSTSTORE_TYPE = CONFIG_PREFIX + "truststore.type";
081  public static final String TLS_CONFIG_TRUSTSTORE_PASSWORD = CONFIG_PREFIX + "truststore.password";
082  public static final String TLS_CONFIG_CLR = CONFIG_PREFIX + "clr";
083  public static final String TLS_CONFIG_OCSP = CONFIG_PREFIX + "ocsp";
084  public static final String TLS_CONFIG_REVERSE_DNS_LOOKUP_ENABLED =
085    CONFIG_PREFIX + "host-verification.reverse-dns.enabled";
086  public static final String TLS_ENABLED_PROTOCOLS = CONFIG_PREFIX + "enabledProtocols";
087  public static final String TLS_CIPHER_SUITES = CONFIG_PREFIX + "ciphersuites";
088  public static final String TLS_CERT_RELOAD = CONFIG_PREFIX + "certReload";
089  public static final String TLS_USE_OPENSSL = CONFIG_PREFIX + "useOpenSsl";
090
091  //
092  // Server-side specific configs
093  //
094  public static final String HBASE_SERVER_NETTY_TLS_ENABLED = "hbase.server.netty.tls.enabled";
095  public static final String HBASE_SERVER_NETTY_TLS_CLIENT_AUTH_MODE =
096    "hbase.server.netty.tls.client.auth.mode";
097  public static final String HBASE_SERVER_NETTY_TLS_VERIFY_CLIENT_HOSTNAME =
098    "hbase.server.netty.tls.verify.client.hostname";
099  public static final String HBASE_SERVER_NETTY_TLS_SUPPORTPLAINTEXT =
100    "hbase.server.netty.tls.supportplaintext";
101
102  /**
103   * Set the SSL wrapSize for netty. This is only a maximum wrap size. Buffers smaller than this
104   * will not be consolidated, but buffers larger than this will be split into multiple wrap
105   * buffers. The netty default of 16k is not great for hbase which tends to return larger payloads
106   * than that, meaning most responses end up getting chunked up. This leads to more memory
107   * contention in netty's PoolArena. See https://github.com/netty/netty/pull/13551
108   */
109  public static final String HBASE_SERVER_NETTY_TLS_WRAP_SIZE = "hbase.server.netty.tls.wrapSize";
110  public static final int DEFAULT_HBASE_SERVER_NETTY_TLS_WRAP_SIZE = 1024 * 1024;
111  //
112  // Client-side specific configs
113  //
114  public static final String HBASE_CLIENT_NETTY_TLS_ENABLED = "hbase.client.netty.tls.enabled";
115  public static final String HBASE_CLIENT_NETTY_TLS_VERIFY_SERVER_HOSTNAME =
116    "hbase.client.netty.tls.verify.server.hostname";
117  public static final String HBASE_CLIENT_NETTY_TLS_HANDSHAKETIMEOUT =
118    "hbase.client.netty.tls.handshaketimeout";
119  public static final int DEFAULT_HANDSHAKE_DETECTION_TIMEOUT_MILLIS = 5000;
120
121  public static final String HBASE_TLS_FILEPOLL_INTERVAL_MILLIS =
122    CONFIG_PREFIX + "filepoll.interval.millis";
123  // 1 minute
124  private static final long DEFAULT_FILE_POLL_INTERVAL = Duration.ofSeconds(60).toMillis();
125
126  /**
127   * Enum specifying the client auth requirement of server-side TLS sockets created by this
128   * X509Util.
129   * <ul>
130   * <li>NONE - do not request a client certificate.</li>
131   * <li>WANT - request a client certificate, but allow anonymous clients to connect.</li>
132   * <li>NEED - require a client certificate, disconnect anonymous clients.</li>
133   * </ul>
134   * If the config property is not set, the default value is NEED.
135   */
136  public enum ClientAuth {
137    NONE(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth.NONE),
138    WANT(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth.OPTIONAL),
139    NEED(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth.REQUIRE);
140
141    private final org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth nettyAuth;
142
143    ClientAuth(org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth nettyAuth) {
144      this.nettyAuth = nettyAuth;
145    }
146
147    /**
148     * Converts a property value to a ClientAuth enum. If the input string is empty or null, returns
149     * <code>ClientAuth.NEED</code>.
150     * @param prop the property string.
151     * @return the ClientAuth.
152     * @throws IllegalArgumentException if the property value is not "NONE", "WANT", "NEED", or
153     *                                  empty/null.
154     */
155    public static ClientAuth fromPropertyValue(String prop) {
156      if (prop == null || prop.length() == 0) {
157        return NEED;
158      }
159      return ClientAuth.valueOf(prop.toUpperCase());
160    }
161
162    public org.apache.hbase.thirdparty.io.netty.handler.ssl.ClientAuth toNettyClientAuth() {
163      return nettyAuth;
164    }
165  }
166
167  private X509Util() {
168    // disabled
169  }
170
171  public static SslContext createSslContextForClient(Configuration config)
172    throws X509Exception, IOException {
173
174    SslContextBuilder sslContextBuilder = SslContextBuilder.forClient();
175
176    configureOpenSslIfAvailable(sslContextBuilder, config);
177    String keyStoreLocation = config.get(TLS_CONFIG_KEYSTORE_LOCATION, "");
178    char[] keyStorePassword = config.getPassword(TLS_CONFIG_KEYSTORE_PASSWORD);
179    String keyStoreType = config.get(TLS_CONFIG_KEYSTORE_TYPE, "");
180
181    if (keyStoreLocation.isEmpty()) {
182      LOG.warn(TLS_CONFIG_KEYSTORE_LOCATION + " not specified");
183    } else {
184      sslContextBuilder
185        .keyManager(createKeyManager(keyStoreLocation, keyStorePassword, keyStoreType));
186    }
187
188    String trustStoreLocation = config.get(TLS_CONFIG_TRUSTSTORE_LOCATION, "");
189    char[] trustStorePassword = config.getPassword(TLS_CONFIG_TRUSTSTORE_PASSWORD);
190    String trustStoreType = config.get(TLS_CONFIG_TRUSTSTORE_TYPE, "");
191
192    boolean sslCrlEnabled = config.getBoolean(TLS_CONFIG_CLR, false);
193    boolean sslOcspEnabled = config.getBoolean(TLS_CONFIG_OCSP, false);
194
195    boolean verifyServerHostname =
196      config.getBoolean(HBASE_CLIENT_NETTY_TLS_VERIFY_SERVER_HOSTNAME, true);
197    boolean allowReverseDnsLookup = config.getBoolean(TLS_CONFIG_REVERSE_DNS_LOOKUP_ENABLED, true);
198
199    if (trustStoreLocation.isEmpty()) {
200      LOG.warn(TLS_CONFIG_TRUSTSTORE_LOCATION + " not specified");
201    } else {
202      sslContextBuilder
203        .trustManager(createTrustManager(trustStoreLocation, trustStorePassword, trustStoreType,
204          sslCrlEnabled, sslOcspEnabled, verifyServerHostname, allowReverseDnsLookup));
205    }
206
207    sslContextBuilder.enableOcsp(sslOcspEnabled);
208    String[] enabledProtocols = getEnabledProtocols(config);
209    if (enabledProtocols != null) {
210      sslContextBuilder.protocols(enabledProtocols);
211    }
212    String[] cipherSuites = getCipherSuites(config);
213    if (cipherSuites != null) {
214      sslContextBuilder.ciphers(Arrays.asList(cipherSuites));
215    }
216
217    return sslContextBuilder.build();
218  }
219
220  /**
221   * Adds SslProvider.OPENSSL if OpenSsl is available and enabled. In order to make it available,
222   * one must ensure that a properly shaded netty-tcnative is on the classpath. Properly shaded
223   * means relocated to be prefixed with "org.apache.hbase.thirdparty" like the rest of the netty
224   * classes. We make available org.apache.hbase:hbase-openssl as a convenience module which one can
225   * use to pull in a shaded netty-tcnative statically linked against boringssl.
226   */
227  private static boolean configureOpenSslIfAvailable(SslContextBuilder sslContextBuilder,
228    Configuration conf) {
229    if (OpenSsl.isAvailable() && conf.getBoolean(TLS_USE_OPENSSL, true)) {
230      LOG.debug("Using netty-tcnative to accelerate SSL handling");
231      sslContextBuilder.sslProvider(SslProvider.OPENSSL);
232      return true;
233    } else {
234      if (LOG.isDebugEnabled()) {
235        LOG.debug("Using default JDK SSL provider because netty-tcnative is not {}",
236          OpenSsl.isAvailable() ? "enabled" : "available");
237      }
238      sslContextBuilder.sslProvider(SslProvider.JDK);
239      return false;
240    }
241  }
242
243  public static SslContext createSslContextForServer(Configuration config)
244    throws X509Exception, IOException {
245    String keyStoreLocation = config.get(TLS_CONFIG_KEYSTORE_LOCATION, "");
246    char[] keyStorePassword = config.getPassword(TLS_CONFIG_KEYSTORE_PASSWORD);
247    String keyStoreType = config.get(TLS_CONFIG_KEYSTORE_TYPE, "");
248
249    if (keyStoreLocation.isEmpty()) {
250      throw new SSLContextException(
251        "Keystore is required for SSL server: " + TLS_CONFIG_KEYSTORE_LOCATION);
252    }
253
254    SslContextBuilder sslContextBuilder;
255    sslContextBuilder = SslContextBuilder
256      .forServer(createKeyManager(keyStoreLocation, keyStorePassword, keyStoreType));
257
258    configureOpenSslIfAvailable(sslContextBuilder, config);
259    String trustStoreLocation = config.get(TLS_CONFIG_TRUSTSTORE_LOCATION, "");
260    char[] trustStorePassword = config.getPassword(TLS_CONFIG_TRUSTSTORE_PASSWORD);
261    String trustStoreType = config.get(TLS_CONFIG_TRUSTSTORE_TYPE, "");
262
263    boolean sslCrlEnabled = config.getBoolean(TLS_CONFIG_CLR, false);
264    boolean sslOcspEnabled = config.getBoolean(TLS_CONFIG_OCSP, false);
265
266    ClientAuth clientAuth =
267      ClientAuth.fromPropertyValue(config.get(HBASE_SERVER_NETTY_TLS_CLIENT_AUTH_MODE));
268    boolean verifyClientHostname =
269      config.getBoolean(HBASE_SERVER_NETTY_TLS_VERIFY_CLIENT_HOSTNAME, true);
270    boolean allowReverseDnsLookup = config.getBoolean(TLS_CONFIG_REVERSE_DNS_LOOKUP_ENABLED, true);
271
272    if (trustStoreLocation.isEmpty()) {
273      LOG.warn(TLS_CONFIG_TRUSTSTORE_LOCATION + " not specified");
274    } else {
275      sslContextBuilder
276        .trustManager(createTrustManager(trustStoreLocation, trustStorePassword, trustStoreType,
277          sslCrlEnabled, sslOcspEnabled, verifyClientHostname, allowReverseDnsLookup));
278    }
279
280    sslContextBuilder.enableOcsp(sslOcspEnabled);
281    String[] enabledProtocols = getEnabledProtocols(config);
282    if (enabledProtocols != null) {
283      sslContextBuilder.protocols(enabledProtocols);
284    }
285    String[] cipherSuites = getCipherSuites(config);
286    if (cipherSuites != null) {
287      sslContextBuilder.ciphers(Arrays.asList(cipherSuites));
288    }
289    sslContextBuilder.clientAuth(clientAuth.toNettyClientAuth());
290
291    return sslContextBuilder.build();
292  }
293
294  /**
295   * Creates a key manager by loading the key store from the given file of the given type,
296   * optionally decrypting it using the given password.
297   * @param keyStoreLocation the location of the key store file.
298   * @param keyStorePassword optional password to decrypt the key store. If empty, assumes the key
299   *                         store is not encrypted.
300   * @param keyStoreType     must be JKS, PEM, PKCS12, BCFKS or null. If null, attempts to
301   *                         autodetect the key store type from the file extension (e.g. .jks /
302   *                         .pem).
303   * @return the key manager.
304   * @throws KeyManagerException if something goes wrong.
305   */
306  static X509KeyManager createKeyManager(String keyStoreLocation, char[] keyStorePassword,
307    String keyStoreType) throws KeyManagerException {
308
309    if (keyStorePassword == null) {
310      keyStorePassword = EMPTY_CHAR_ARRAY;
311    }
312
313    try {
314      KeyStoreFileType storeFileType =
315        KeyStoreFileType.fromPropertyValueOrFileName(keyStoreType, keyStoreLocation);
316      KeyStore ks = FileKeyStoreLoaderBuilderProvider.getBuilderForKeyStoreFileType(storeFileType)
317        .setKeyStorePath(keyStoreLocation).setKeyStorePassword(keyStorePassword).build()
318        .loadKeyStore();
319
320      KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX");
321      kmf.init(ks, keyStorePassword);
322
323      for (KeyManager km : kmf.getKeyManagers()) {
324        if (km instanceof X509KeyManager) {
325          return (X509KeyManager) km;
326        }
327      }
328      throw new KeyManagerException("Couldn't find X509KeyManager");
329    } catch (IOException | GeneralSecurityException | IllegalArgumentException e) {
330      throw new KeyManagerException(e);
331    }
332  }
333
334  /**
335   * Creates a trust manager by loading the trust store from the given file of the given type,
336   * optionally decrypting it using the given password.
337   * @param trustStoreLocation    the location of the trust store file.
338   * @param trustStorePassword    optional password to decrypt the trust store (only applies to JKS
339   *                              trust stores). If empty, assumes the trust store is not encrypted.
340   * @param trustStoreType        must be JKS, PEM, PKCS12, BCFKS or null. If null, attempts to
341   *                              autodetect the trust store type from the file extension (e.g. .jks
342   *                              / .pem).
343   * @param crlEnabled            enable CRL (certificate revocation list) checks.
344   * @param ocspEnabled           enable OCSP (online certificate status protocol) checks.
345   * @param verifyHostName        if true, ssl peer hostname must match name in certificate
346   * @param allowReverseDnsLookup if true, allow falling back to reverse dns lookup in verifying
347   *                              hostname
348   * @return the trust manager.
349   * @throws TrustManagerException if something goes wrong.
350   */
351  static X509TrustManager createTrustManager(String trustStoreLocation, char[] trustStorePassword,
352    String trustStoreType, boolean crlEnabled, boolean ocspEnabled, boolean verifyHostName,
353    boolean allowReverseDnsLookup) throws TrustManagerException {
354
355    if (trustStorePassword == null) {
356      trustStorePassword = EMPTY_CHAR_ARRAY;
357    }
358
359    try {
360      KeyStoreFileType storeFileType =
361        KeyStoreFileType.fromPropertyValueOrFileName(trustStoreType, trustStoreLocation);
362      KeyStore ts = FileKeyStoreLoaderBuilderProvider.getBuilderForKeyStoreFileType(storeFileType)
363        .setTrustStorePath(trustStoreLocation).setTrustStorePassword(trustStorePassword).build()
364        .loadTrustStore();
365
366      PKIXBuilderParameters pbParams = new PKIXBuilderParameters(ts, new X509CertSelector());
367      if (crlEnabled || ocspEnabled) {
368        pbParams.setRevocationEnabled(true);
369        System.setProperty("com.sun.net.ssl.checkRevocation", "true");
370        if (crlEnabled) {
371          System.setProperty("com.sun.security.enableCRLDP", "true");
372        }
373        if (ocspEnabled) {
374          Security.setProperty("ocsp.enable", "true");
375        }
376      } else {
377        pbParams.setRevocationEnabled(false);
378      }
379
380      // Revocation checking is only supported with the PKIX algorithm
381      TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX");
382      tmf.init(new CertPathTrustManagerParameters(pbParams));
383
384      for (final TrustManager tm : tmf.getTrustManagers()) {
385        if (tm instanceof X509ExtendedTrustManager) {
386          return new HBaseTrustManager((X509ExtendedTrustManager) tm, verifyHostName,
387            allowReverseDnsLookup);
388        }
389      }
390      throw new TrustManagerException("Couldn't find X509TrustManager");
391    } catch (IOException | GeneralSecurityException | IllegalArgumentException e) {
392      throw new TrustManagerException(e);
393    }
394  }
395
396  private static String[] getEnabledProtocols(Configuration config) {
397    String enabledProtocolsInput = config.get(TLS_ENABLED_PROTOCOLS);
398    if (enabledProtocolsInput == null) {
399      enabledProtocolsInput = config.get(TLS_CONFIG_PROTOCOL);
400    }
401    if (enabledProtocolsInput != null) {
402      return enabledProtocolsInput.split(",");
403    } else {
404      return null;
405    }
406  }
407
408  private static String[] getCipherSuites(Configuration config) {
409    String cipherSuitesInput = config.get(TLS_CIPHER_SUITES);
410    if (cipherSuitesInput == null) {
411      return null;
412    } else {
413      return cipherSuitesInput.split(",");
414    }
415  }
416
417  /**
418   * Enable certificate file reloading by creating FileWatchers for keystore and truststore.
419   * AtomicReferences will be set with the new instances. resetContext - if not null - will be
420   * called when the file has been modified.
421   * @param keystoreWatcher   Reference to keystoreFileWatcher.
422   * @param trustStoreWatcher Reference to truststoreFileWatcher.
423   * @param resetContext      Callback for file changes.
424   */
425  public static void enableCertFileReloading(Configuration config,
426    AtomicReference<FileChangeWatcher> keystoreWatcher,
427    AtomicReference<FileChangeWatcher> trustStoreWatcher, Runnable resetContext)
428    throws IOException {
429    String keyStoreLocation = config.get(TLS_CONFIG_KEYSTORE_LOCATION, "");
430    keystoreWatcher.set(newFileChangeWatcher(config, keyStoreLocation, resetContext));
431    String trustStoreLocation = config.get(TLS_CONFIG_TRUSTSTORE_LOCATION, "");
432    // we are using the same callback for both. there's no reason to kick off two
433    // threads if keystore/truststore are both at the same location
434    if (!keyStoreLocation.equals(trustStoreLocation)) {
435      trustStoreWatcher.set(newFileChangeWatcher(config, trustStoreLocation, resetContext));
436    }
437  }
438
439  private static FileChangeWatcher newFileChangeWatcher(Configuration config, String fileLocation,
440    Runnable resetContext) throws IOException {
441    if (fileLocation == null || fileLocation.isEmpty() || resetContext == null) {
442      return null;
443    }
444    final Path filePath = Paths.get(fileLocation).toAbsolutePath();
445    FileChangeWatcher fileChangeWatcher =
446      new FileChangeWatcher(filePath, Objects.toString(filePath.getFileName()),
447        Duration
448          .ofMillis(config.getLong(HBASE_TLS_FILEPOLL_INTERVAL_MILLIS, DEFAULT_FILE_POLL_INTERVAL)),
449        watchEventFilePath -> handleWatchEvent(watchEventFilePath, resetContext));
450    fileChangeWatcher.start();
451    return fileChangeWatcher;
452  }
453
454  /**
455   * Handler for watch events that let us know a file we may care about has changed on disk.
456   */
457  private static void handleWatchEvent(Path filePath, Runnable resetContext) {
458    LOG.info("Attempting to reset default SSL context after receiving watch event on file {}",
459      filePath);
460    resetContext.run();
461  }
462}