Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNull;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023import static org.junit.jupiter.api.Assertions.fail;
024
025import java.io.File;
026import java.io.IOException;
027import java.lang.reflect.InvocationTargetException;
028import java.lang.reflect.Method;
029import java.util.List;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.hbase.security.User;
032import org.apache.hadoop.hbase.testclassification.MiscTests;
033import org.apache.hadoop.hbase.testclassification.SmallTests;
034import org.junit.jupiter.api.AfterAll;
035import org.junit.jupiter.api.Tag;
036import org.junit.jupiter.api.Test;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
041
042@Tag(MiscTests.TAG)
043@Tag(SmallTests.TAG)
044public class TestHBaseConfiguration {
045
046  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseConfiguration.class);
047
048  private static HBaseCommonTestingUtil UTIL = new HBaseCommonTestingUtil();
049
050  @AfterAll
051  public static void tearDown() throws IOException {
052    UTIL.cleanupTestDir();
053  }
054
055  @Test
056  public void testSubset() {
057    Configuration conf = HBaseConfiguration.create();
058    // subset is used in TableMapReduceUtil#initCredentials to support different security
059    // configurations between source and destination clusters, so we'll use that as an example
060    String prefix = "hbase.mapred.output.";
061    conf.set("hbase.security.authentication", "kerberos");
062    conf.set("hbase.regionserver.kerberos.principal", "hbasesource");
063    HBaseConfiguration.setWithPrefix(conf, prefix, ImmutableMap
064      .of("hbase.regionserver.kerberos.principal", "hbasedest", "", "shouldbemissing").entrySet());
065
066    Configuration subsetConf = HBaseConfiguration.subset(conf, prefix);
067    assertNull(subsetConf.get(prefix + "hbase.regionserver.kerberos.principal"));
068    assertEquals("hbasedest", subsetConf.get("hbase.regionserver.kerberos.principal"));
069    assertNull(subsetConf.get("hbase.security.authentication"));
070    assertNull(subsetConf.get(""));
071
072    Configuration mergedConf = HBaseConfiguration.create(conf);
073    HBaseConfiguration.merge(mergedConf, subsetConf);
074
075    assertEquals("hbasedest", mergedConf.get("hbase.regionserver.kerberos.principal"));
076    assertEquals("kerberos", mergedConf.get("hbase.security.authentication"));
077    assertEquals("shouldbemissing", mergedConf.get(prefix));
078  }
079
080  @Test
081  public void testGetPassword() throws Exception {
082    Configuration conf = HBaseConfiguration.create();
083    conf.set(ReflectiveCredentialProviderClient.CREDENTIAL_PROVIDER_PATH, "jceks://file"
084      + new File(UTIL.getDataTestDir().toUri().getPath(), "foo.jks").getCanonicalPath());
085    ReflectiveCredentialProviderClient client = new ReflectiveCredentialProviderClient();
086    if (client.isHadoopCredentialProviderAvailable()) {
087      char[] keyPass = { 'k', 'e', 'y', 'p', 'a', 's', 's' };
088      char[] storePass = { 's', 't', 'o', 'r', 'e', 'p', 'a', 's', 's' };
089      client.createEntry(conf, "ssl.keypass.alias", keyPass);
090      client.createEntry(conf, "ssl.storepass.alias", storePass);
091
092      String keypass = HBaseConfiguration.getPassword(conf, "ssl.keypass.alias", null);
093      assertEquals(keypass, new String(keyPass));
094
095      String storepass = HBaseConfiguration.getPassword(conf, "ssl.storepass.alias", null);
096      assertEquals(storepass, new String(storePass));
097    }
098  }
099
100  @Test
101  public void testSecurityConfCaseInsensitive() {
102    Configuration conf = HBaseConfiguration.create();
103    conf.set("hbase.security.authentication", "kerberos");
104    assertTrue(User.isHBaseSecurityEnabled(conf));
105
106    conf.set("hbase.security.authentication", "KERBEROS");
107    assertTrue(User.isHBaseSecurityEnabled(conf));
108
109    conf.set("hbase.security.authentication", "KERBeros");
110    assertTrue(User.isHBaseSecurityEnabled(conf));
111  }
112
113  @Test
114  public void testGetConfigOfShortcircuitRead() throws Exception {
115    Configuration conf = HBaseConfiguration.create();
116    Configuration.addDefaultResource("hdfs-scr-disabled.xml");
117    assertEquals("hdfs-scr-disabled.xml",
118      conf.getPropertySources("dfs.client.read.shortcircuit")[0]);
119    assertEquals("false", conf.get("dfs.client.read.shortcircuit"));
120    assertNull(conf.get("dfs.domain.socket.path"));
121    Configuration.addDefaultResource("hdfs-scr-enabled.xml");
122    assertEquals("hdfs-scr-enabled.xml",
123      conf.getPropertySources("dfs.client.read.shortcircuit")[0]);
124    assertEquals("hdfs-scr-enabled.xml", conf.getPropertySources("dfs.domain.socket.path")[0]);
125    assertEquals("true", conf.get("dfs.client.read.shortcircuit"));
126    assertEquals("/var/lib/hadoop-hdfs/dn_socket", conf.get("dfs.domain.socket.path"));
127  }
128
129  @Test
130  public void testDeprecatedConfigurations() {
131    // Configuration.addDeprecations before create Configuration object
132    Configuration.addDeprecations(new Configuration.DeprecationDelta[] {
133      new Configuration.DeprecationDelta("hbase.deprecated.conf", "hbase.new.conf"),
134      new Configuration.DeprecationDelta("hbase.deprecated.conf2", "hbase.new.conf2") });
135    Configuration conf = HBaseConfiguration.create();
136    conf.addResource("hbase-deprecated-conf.xml");
137    assertEquals("1000", conf.get("hbase.new.conf"));
138    assertEquals("1000", conf.get("hbase.new.conf2"));
139  }
140
141  private static class ReflectiveCredentialProviderClient {
142    public static final String HADOOP_CRED_PROVIDER_FACTORY_CLASS_NAME =
143      "org.apache.hadoop.security.alias.JavaKeyStoreProvider$Factory";
144    public static final String HADOOP_CRED_PROVIDER_FACTORY_GET_PROVIDERS_METHOD_NAME =
145      "getProviders";
146
147    public static final String HADOOP_CRED_PROVIDER_CLASS_NAME =
148      "org.apache.hadoop.security.alias.CredentialProvider";
149    public static final String HADOOP_CRED_PROVIDER_GET_CREDENTIAL_ENTRY_METHOD_NAME =
150      "getCredentialEntry";
151    public static final String HADOOP_CRED_PROVIDER_GET_ALIASES_METHOD_NAME = "getAliases";
152    public static final String HADOOP_CRED_PROVIDER_CREATE_CREDENTIAL_ENTRY_METHOD_NAME =
153      "createCredentialEntry";
154    public static final String HADOOP_CRED_PROVIDER_FLUSH_METHOD_NAME = "flush";
155
156    public static final String HADOOP_CRED_ENTRY_CLASS_NAME =
157      "org.apache.hadoop.security.alias.CredentialProvider$CredentialEntry";
158    public static final String HADOOP_CRED_ENTRY_GET_CREDENTIAL_METHOD_NAME = "getCredential";
159
160    public static final String CREDENTIAL_PROVIDER_PATH =
161      "hadoop.security.credential.provider.path";
162
163    private static Object hadoopCredProviderFactory = null;
164    private static Method getProvidersMethod = null;
165    private static Method getCredentialEntryMethod = null;
166    private static Method getCredentialMethod = null;
167    private static Method createCredentialEntryMethod = null;
168    private static Method flushMethod = null;
169    private static Boolean hadoopClassesAvailable = null;
170
171    /**
172     * Determine if we can load the necessary CredentialProvider classes. Only loaded the first
173     * time, so subsequent invocations of this method should return fast.
174     * @return True if the CredentialProvider classes/methods are available, false otherwise.
175     */
176    private boolean isHadoopCredentialProviderAvailable() {
177      if (null != hadoopClassesAvailable) {
178        // Make sure everything is initialized as expected
179        if (
180          hadoopClassesAvailable && null != getProvidersMethod && null != hadoopCredProviderFactory
181            && null != getCredentialEntryMethod && null != getCredentialMethod
182        ) {
183          return true;
184        } else {
185          // Otherwise we failed to load it
186          return false;
187        }
188      }
189
190      hadoopClassesAvailable = false;
191
192      // Load Hadoop CredentialProviderFactory
193      Class<?> hadoopCredProviderFactoryClz;
194      try {
195        hadoopCredProviderFactoryClz = Class.forName(HADOOP_CRED_PROVIDER_FACTORY_CLASS_NAME);
196      } catch (ClassNotFoundException e) {
197        return false;
198      }
199      // Instantiate Hadoop CredentialProviderFactory
200      try {
201        hadoopCredProviderFactory =
202          hadoopCredProviderFactoryClz.getDeclaredConstructor().newInstance();
203      } catch (Exception e) {
204        return false;
205      }
206
207      try {
208        getProvidersMethod = loadMethod(hadoopCredProviderFactoryClz,
209          HADOOP_CRED_PROVIDER_FACTORY_GET_PROVIDERS_METHOD_NAME, Configuration.class);
210        // Load Hadoop CredentialProvider
211        Class<?> hadoopCredProviderClz;
212        hadoopCredProviderClz = Class.forName(HADOOP_CRED_PROVIDER_CLASS_NAME);
213        getCredentialEntryMethod = loadMethod(hadoopCredProviderClz,
214          HADOOP_CRED_PROVIDER_GET_CREDENTIAL_ENTRY_METHOD_NAME, String.class);
215
216        createCredentialEntryMethod = loadMethod(hadoopCredProviderClz,
217          HADOOP_CRED_PROVIDER_CREATE_CREDENTIAL_ENTRY_METHOD_NAME, String.class, char[].class);
218
219        flushMethod = loadMethod(hadoopCredProviderClz, HADOOP_CRED_PROVIDER_FLUSH_METHOD_NAME);
220
221        // Load Hadoop CredentialEntry
222        Class<?> hadoopCredentialEntryClz;
223        try {
224          hadoopCredentialEntryClz = Class.forName(HADOOP_CRED_ENTRY_CLASS_NAME);
225        } catch (ClassNotFoundException e) {
226          LOG.error("Failed to load class:" + e);
227          return false;
228        }
229
230        getCredentialMethod =
231          loadMethod(hadoopCredentialEntryClz, HADOOP_CRED_ENTRY_GET_CREDENTIAL_METHOD_NAME);
232      } catch (Exception e1) {
233        return false;
234      }
235
236      hadoopClassesAvailable = true;
237      LOG.info("Credential provider classes have been"
238        + " loaded and initialized successfully through reflection.");
239      return true;
240    }
241
242    private Method loadMethod(Class<?> clz, String name, Class<?>... classes) throws Exception {
243      Method method;
244      try {
245        method = clz.getMethod(name, classes);
246      } catch (SecurityException e) {
247        fail("security exception caught for: " + name + " in " + clz.getCanonicalName());
248        throw e;
249      } catch (NoSuchMethodException e) {
250        LOG.error("Failed to load the " + name + ": " + e);
251        fail("no such method: " + name + " in " + clz.getCanonicalName());
252        throw e;
253      }
254      return method;
255    }
256
257    /**
258     * Wrapper to fetch the configured {@code List<CredentialProvider>}s. Configuration with
259     * GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS defined
260     * @return List of CredentialProviders, or null if they could not be loaded
261     */
262    @SuppressWarnings("unchecked")
263    protected List<Object> getCredentialProviders(Configuration conf) {
264      // Call CredentialProviderFactory.getProviders(Configuration)
265      Object providersObj;
266      try {
267        providersObj = getProvidersMethod.invoke(hadoopCredProviderFactory, conf);
268      } catch (IllegalArgumentException | InvocationTargetException | IllegalAccessException e) {
269        LOG.error("Failed to invoke: " + getProvidersMethod.getName() + ": " + e);
270        return null;
271      }
272
273      // Cast the Object to List<Object> (actually List<CredentialProvider>)
274      try {
275        return (List<Object>) providersObj;
276      } catch (ClassCastException e) {
277        return null;
278      }
279    }
280
281    /**
282     * Create a CredentialEntry using the configured Providers. If multiple CredentialProviders are
283     * configured, the first will be used. Configuration for the CredentialProvider CredentialEntry
284     * name (alias) The credential
285     */
286    public void createEntry(Configuration conf, String name, char[] credential) throws Exception {
287      if (!isHadoopCredentialProviderAvailable()) {
288        return;
289      }
290
291      List<Object> providers = getCredentialProviders(conf);
292      if (null == providers) {
293        throw new IOException(
294          "Could not fetch any CredentialProviders, " + "is the implementation available?");
295      }
296
297      Object provider = providers.get(0);
298      createEntryInProvider(provider, name, credential);
299    }
300
301    /**
302     * Create a CredentialEntry with the give name and credential in the credentialProvider. The
303     * credentialProvider argument must be an instance of Hadoop CredentialProvider. Instance of
304     * CredentialProvider CredentialEntry name (alias) The credential to store
305     */
306    private void createEntryInProvider(Object credentialProvider, String name, char[] credential)
307      throws Exception {
308      if (!isHadoopCredentialProviderAvailable()) {
309        return;
310      }
311
312      try {
313        createCredentialEntryMethod.invoke(credentialProvider, name, credential);
314      } catch (IllegalArgumentException | InvocationTargetException | IllegalAccessException e) {
315        return;
316      }
317
318      flushMethod.invoke(credentialProvider);
319    }
320  }
321}