Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022import static org.junit.Assert.fail;
023
024import java.io.File;
025import java.io.IOException;
026import java.lang.reflect.InvocationTargetException;
027import java.lang.reflect.Method;
028import java.util.List;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.security.User;
031import org.apache.hadoop.hbase.testclassification.MiscTests;
032import org.apache.hadoop.hbase.testclassification.SmallTests;
033import org.junit.AfterClass;
034import org.junit.Assert;
035import org.junit.ClassRule;
036import org.junit.Test;
037import org.junit.experimental.categories.Category;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
042
043@Category({ MiscTests.class, SmallTests.class })
044public class TestHBaseConfiguration {
045  @ClassRule
046  public static final HBaseClassTestRule CLASS_RULE =
047    HBaseClassTestRule.forClass(TestHBaseConfiguration.class);
048
049  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseConfiguration.class);
050
051  private static HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility();
052
053  @AfterClass
054  public static void tearDown() throws IOException {
055    UTIL.cleanupTestDir();
056  }
057
058  @Test
059  public void testSubset() {
060    Configuration conf = HBaseConfiguration.create();
061    // subset is used in TableMapReduceUtil#initCredentials to support different security
062    // configurations between source and destination clusters, so we'll use that as an example
063    String prefix = "hbase.mapred.output.";
064    conf.set("hbase.security.authentication", "kerberos");
065    conf.set("hbase.regionserver.kerberos.principal", "hbasesource");
066    HBaseConfiguration.setWithPrefix(conf, prefix, ImmutableMap
067      .of("hbase.regionserver.kerberos.principal", "hbasedest", "", "shouldbemissing").entrySet());
068
069    Configuration subsetConf = HBaseConfiguration.subset(conf, prefix);
070    assertNull(subsetConf.get(prefix + "hbase.regionserver.kerberos.principal"));
071    assertEquals("hbasedest", subsetConf.get("hbase.regionserver.kerberos.principal"));
072    assertNull(subsetConf.get("hbase.security.authentication"));
073    assertNull(subsetConf.get(""));
074
075    Configuration mergedConf = HBaseConfiguration.create(conf);
076    HBaseConfiguration.merge(mergedConf, subsetConf);
077
078    assertEquals("hbasedest", mergedConf.get("hbase.regionserver.kerberos.principal"));
079    assertEquals("kerberos", mergedConf.get("hbase.security.authentication"));
080    assertEquals("shouldbemissing", mergedConf.get(prefix));
081  }
082
083  @Test
084  public void testGetPassword() throws Exception {
085    Configuration conf = HBaseConfiguration.create();
086    conf.set(ReflectiveCredentialProviderClient.CREDENTIAL_PROVIDER_PATH, "jceks://file"
087      + new File(UTIL.getDataTestDir().toUri().getPath(), "foo.jks").getCanonicalPath());
088    ReflectiveCredentialProviderClient client = new ReflectiveCredentialProviderClient();
089    if (client.isHadoopCredentialProviderAvailable()) {
090      char[] keyPass = { 'k', 'e', 'y', 'p', 'a', 's', 's' };
091      char[] storePass = { 's', 't', 'o', 'r', 'e', 'p', 'a', 's', 's' };
092      client.createEntry(conf, "ssl.keypass.alias", keyPass);
093      client.createEntry(conf, "ssl.storepass.alias", storePass);
094
095      String keypass = HBaseConfiguration.getPassword(conf, "ssl.keypass.alias", null);
096      assertEquals(keypass, new String(keyPass));
097
098      String storepass = HBaseConfiguration.getPassword(conf, "ssl.storepass.alias", null);
099      assertEquals(storepass, new String(storePass));
100    }
101  }
102
103  @Test
104  public void testSecurityConfCaseInsensitive() {
105    Configuration conf = HBaseConfiguration.create();
106    conf.set("hbase.security.authentication", "kerberos");
107    Assert.assertTrue(User.isHBaseSecurityEnabled(conf));
108
109    conf.set("hbase.security.authentication", "KERBEROS");
110    Assert.assertTrue(User.isHBaseSecurityEnabled(conf));
111
112    conf.set("hbase.security.authentication", "KERBeros");
113    Assert.assertTrue(User.isHBaseSecurityEnabled(conf));
114  }
115
116  @Test
117  public void testGetConfigOfShortcircuitRead() throws Exception {
118    Configuration conf = HBaseConfiguration.create();
119    Configuration.addDefaultResource("hdfs-scr-disabled.xml");
120    assertEquals("hdfs-scr-disabled.xml",
121      conf.getPropertySources("dfs.client.read.shortcircuit")[0]);
122    assertEquals("false", conf.get("dfs.client.read.shortcircuit"));
123    assertNull(conf.get("dfs.domain.socket.path"));
124    Configuration.addDefaultResource("hdfs-scr-enabled.xml");
125    assertEquals("hdfs-scr-enabled.xml",
126      conf.getPropertySources("dfs.client.read.shortcircuit")[0]);
127    assertEquals("hdfs-scr-enabled.xml", conf.getPropertySources("dfs.domain.socket.path")[0]);
128    assertEquals("true", conf.get("dfs.client.read.shortcircuit"));
129    assertEquals("/var/lib/hadoop-hdfs/dn_socket", conf.get("dfs.domain.socket.path"));
130  }
131
132  @Test
133  public void testDeprecatedConfigurations() {
134    // Configuration.addDeprecations before create Configuration object
135    Configuration.addDeprecations(new Configuration.DeprecationDelta[] {
136      new Configuration.DeprecationDelta("hbase.deprecated.conf", "hbase.new.conf"),
137      new Configuration.DeprecationDelta("hbase.deprecated.conf2", "hbase.new.conf2") });
138    Configuration conf = HBaseConfiguration.create();
139    conf.addResource("hbase-deprecated-conf.xml");
140    assertEquals("1000", conf.get("hbase.new.conf"));
141    assertEquals("1000", conf.get("hbase.new.conf2"));
142  }
143
144  private static class ReflectiveCredentialProviderClient {
145    public static final String HADOOP_CRED_PROVIDER_FACTORY_CLASS_NAME =
146      "org.apache.hadoop.security.alias.JavaKeyStoreProvider$Factory";
147    public static final String HADOOP_CRED_PROVIDER_FACTORY_GET_PROVIDERS_METHOD_NAME =
148      "getProviders";
149
150    public static final String HADOOP_CRED_PROVIDER_CLASS_NAME =
151      "org.apache.hadoop.security.alias.CredentialProvider";
152    public static final String HADOOP_CRED_PROVIDER_GET_CREDENTIAL_ENTRY_METHOD_NAME =
153      "getCredentialEntry";
154    public static final String HADOOP_CRED_PROVIDER_GET_ALIASES_METHOD_NAME = "getAliases";
155    public static final String HADOOP_CRED_PROVIDER_CREATE_CREDENTIAL_ENTRY_METHOD_NAME =
156      "createCredentialEntry";
157    public static final String HADOOP_CRED_PROVIDER_FLUSH_METHOD_NAME = "flush";
158
159    public static final String HADOOP_CRED_ENTRY_CLASS_NAME =
160      "org.apache.hadoop.security.alias.CredentialProvider$CredentialEntry";
161    public static final String HADOOP_CRED_ENTRY_GET_CREDENTIAL_METHOD_NAME = "getCredential";
162
163    public static final String CREDENTIAL_PROVIDER_PATH =
164      "hadoop.security.credential.provider.path";
165
166    private static Object hadoopCredProviderFactory = null;
167    private static Method getProvidersMethod = null;
168    private static Method getCredentialEntryMethod = null;
169    private static Method getCredentialMethod = null;
170    private static Method createCredentialEntryMethod = null;
171    private static Method flushMethod = null;
172    private static Boolean hadoopClassesAvailable = null;
173
174    /**
175     * Determine if we can load the necessary CredentialProvider classes. Only loaded the first
176     * time, so subsequent invocations of this method should return fast.
177     * @return True if the CredentialProvider classes/methods are available, false otherwise.
178     */
179    private boolean isHadoopCredentialProviderAvailable() {
180      if (null != hadoopClassesAvailable) {
181        // Make sure everything is initialized as expected
182        if (
183          hadoopClassesAvailable && null != getProvidersMethod && null != hadoopCredProviderFactory
184            && null != getCredentialEntryMethod && null != getCredentialMethod
185        ) {
186          return true;
187        } else {
188          // Otherwise we failed to load it
189          return false;
190        }
191      }
192
193      hadoopClassesAvailable = false;
194
195      // Load Hadoop CredentialProviderFactory
196      Class<?> hadoopCredProviderFactoryClz;
197      try {
198        hadoopCredProviderFactoryClz = Class.forName(HADOOP_CRED_PROVIDER_FACTORY_CLASS_NAME);
199      } catch (ClassNotFoundException e) {
200        return false;
201      }
202      // Instantiate Hadoop CredentialProviderFactory
203      try {
204        hadoopCredProviderFactory =
205          hadoopCredProviderFactoryClz.getDeclaredConstructor().newInstance();
206      } catch (Exception e) {
207        return false;
208      }
209
210      try {
211        getProvidersMethod = loadMethod(hadoopCredProviderFactoryClz,
212          HADOOP_CRED_PROVIDER_FACTORY_GET_PROVIDERS_METHOD_NAME, Configuration.class);
213        // Load Hadoop CredentialProvider
214        Class<?> hadoopCredProviderClz;
215        hadoopCredProviderClz = Class.forName(HADOOP_CRED_PROVIDER_CLASS_NAME);
216        getCredentialEntryMethod = loadMethod(hadoopCredProviderClz,
217          HADOOP_CRED_PROVIDER_GET_CREDENTIAL_ENTRY_METHOD_NAME, String.class);
218
219        createCredentialEntryMethod = loadMethod(hadoopCredProviderClz,
220          HADOOP_CRED_PROVIDER_CREATE_CREDENTIAL_ENTRY_METHOD_NAME, String.class, char[].class);
221
222        flushMethod = loadMethod(hadoopCredProviderClz, HADOOP_CRED_PROVIDER_FLUSH_METHOD_NAME);
223
224        // Load Hadoop CredentialEntry
225        Class<?> hadoopCredentialEntryClz;
226        try {
227          hadoopCredentialEntryClz = Class.forName(HADOOP_CRED_ENTRY_CLASS_NAME);
228        } catch (ClassNotFoundException e) {
229          LOG.error("Failed to load class:" + e);
230          return false;
231        }
232
233        getCredentialMethod =
234          loadMethod(hadoopCredentialEntryClz, HADOOP_CRED_ENTRY_GET_CREDENTIAL_METHOD_NAME);
235      } catch (Exception e1) {
236        return false;
237      }
238
239      hadoopClassesAvailable = true;
240      LOG.info("Credential provider classes have been"
241        + " loaded and initialized successfully through reflection.");
242      return true;
243    }
244
245    private Method loadMethod(Class<?> clz, String name, Class<?>... classes) throws Exception {
246      Method method;
247      try {
248        method = clz.getMethod(name, classes);
249      } catch (SecurityException e) {
250        fail("security exception caught for: " + name + " in " + clz.getCanonicalName());
251        throw e;
252      } catch (NoSuchMethodException e) {
253        LOG.error("Failed to load the " + name + ": " + e);
254        fail("no such method: " + name + " in " + clz.getCanonicalName());
255        throw e;
256      }
257      return method;
258    }
259
260    /**
261     * Wrapper to fetch the configured {@code List<CredentialProvider>}s. n * Configuration with
262     * GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS defined
263     * @return List of CredentialProviders, or null if they could not be loaded
264     */
265    @SuppressWarnings("unchecked")
266    protected List<Object> getCredentialProviders(Configuration conf) {
267      // Call CredentialProviderFactory.getProviders(Configuration)
268      Object providersObj;
269      try {
270        providersObj = getProvidersMethod.invoke(hadoopCredProviderFactory, conf);
271      } catch (IllegalArgumentException | InvocationTargetException | IllegalAccessException e) {
272        LOG.error("Failed to invoke: " + getProvidersMethod.getName() + ": " + e);
273        return null;
274      }
275
276      // Cast the Object to List<Object> (actually List<CredentialProvider>)
277      try {
278        return (List<Object>) providersObj;
279      } catch (ClassCastException e) {
280        return null;
281      }
282    }
283
284    /**
285     * Create a CredentialEntry using the configured Providers. If multiple CredentialProviders are
286     * configured, the first will be used. n * Configuration for the CredentialProvider n *
287     * CredentialEntry name (alias) n * The credential
288     */
289    public void createEntry(Configuration conf, String name, char[] credential) throws Exception {
290      if (!isHadoopCredentialProviderAvailable()) {
291        return;
292      }
293
294      List<Object> providers = getCredentialProviders(conf);
295      if (null == providers) {
296        throw new IOException(
297          "Could not fetch any CredentialProviders, " + "is the implementation available?");
298      }
299
300      Object provider = providers.get(0);
301      createEntryInProvider(provider, name, credential);
302    }
303
304    /**
305     * Create a CredentialEntry with the give name and credential in the credentialProvider. The
306     * credentialProvider argument must be an instance of Hadoop CredentialProvider. n * Instance of
307     * CredentialProvider n * CredentialEntry name (alias) n * The credential to store
308     */
309    private void createEntryInProvider(Object credentialProvider, String name, char[] credential)
310      throws Exception {
311      if (!isHadoopCredentialProviderAvailable()) {
312        return;
313      }
314
315      try {
316        createCredentialEntryMethod.invoke(credentialProvider, name, credential);
317      } catch (IllegalArgumentException | InvocationTargetException | IllegalAccessException e) {
318        return;
319      }
320
321      flushMethod.invoke(credentialProvider);
322    }
323  }
324}