001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022import static org.junit.Assert.fail;
023
024import java.io.File;
025import java.io.IOException;
026import java.lang.reflect.InvocationTargetException;
027import java.lang.reflect.Method;
028import java.util.List;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.security.User;
031import org.apache.hadoop.hbase.testclassification.MiscTests;
032import org.apache.hadoop.hbase.testclassification.SmallTests;
033import org.junit.AfterClass;
034import org.junit.Assert;
035import org.junit.ClassRule;
036import org.junit.Test;
037import org.junit.experimental.categories.Category;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
042
043@Category({MiscTests.class, SmallTests.class})
044public class TestHBaseConfiguration {
045  @ClassRule
046  public static final HBaseClassTestRule CLASS_RULE =
047      HBaseClassTestRule.forClass(TestHBaseConfiguration.class);
048
049  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseConfiguration.class);
050
051  private static HBaseCommonTestingUtil UTIL = new HBaseCommonTestingUtil();
052
053  @AfterClass
054  public static void tearDown() throws IOException {
055    UTIL.cleanupTestDir();
056  }
057
058  @Test
059  public void testSubset() {
060    Configuration conf = HBaseConfiguration.create();
061    // subset is used in TableMapReduceUtil#initCredentials to support different security
062    // configurations between source and destination clusters, so we'll use that as an example
063    String prefix = "hbase.mapred.output.";
064    conf.set("hbase.security.authentication", "kerberos");
065    conf.set("hbase.regionserver.kerberos.principal", "hbasesource");
066    HBaseConfiguration.setWithPrefix(conf, prefix,
067        ImmutableMap.of(
068            "hbase.regionserver.kerberos.principal", "hbasedest",
069            "", "shouldbemissing")
070            .entrySet());
071
072    Configuration subsetConf = HBaseConfiguration.subset(conf, prefix);
073    assertNull(subsetConf.get(prefix + "hbase.regionserver.kerberos.principal"));
074    assertEquals("hbasedest", subsetConf.get("hbase.regionserver.kerberos.principal"));
075    assertNull(subsetConf.get("hbase.security.authentication"));
076    assertNull(subsetConf.get(""));
077
078    Configuration mergedConf = HBaseConfiguration.create(conf);
079    HBaseConfiguration.merge(mergedConf, subsetConf);
080
081    assertEquals("hbasedest", mergedConf.get("hbase.regionserver.kerberos.principal"));
082    assertEquals("kerberos", mergedConf.get("hbase.security.authentication"));
083    assertEquals("shouldbemissing", mergedConf.get(prefix));
084  }
085
086  @Test
087  public void testGetPassword() throws Exception {
088    Configuration conf = HBaseConfiguration.create();
089    conf.set(ReflectiveCredentialProviderClient.CREDENTIAL_PROVIDER_PATH, "jceks://file"
090        + new File(UTIL.getDataTestDir().toUri().getPath(), "foo.jks").getCanonicalPath());
091    ReflectiveCredentialProviderClient client = new ReflectiveCredentialProviderClient();
092    if (client.isHadoopCredentialProviderAvailable()) {
093      char[] keyPass = { 'k', 'e', 'y', 'p', 'a', 's', 's' };
094      char[] storePass = { 's', 't', 'o', 'r', 'e', 'p', 'a', 's', 's' };
095      client.createEntry(conf, "ssl.keypass.alias", keyPass);
096      client.createEntry(conf, "ssl.storepass.alias", storePass);
097
098      String keypass = HBaseConfiguration.getPassword(conf, "ssl.keypass.alias", null);
099      assertEquals(keypass, new String(keyPass));
100
101      String storepass = HBaseConfiguration.getPassword(conf, "ssl.storepass.alias", null);
102      assertEquals(storepass, new String(storePass));
103    }
104  }
105
106  @Test
107  public void testSecurityConfCaseInsensitive() {
108    Configuration conf = HBaseConfiguration.create();
109    conf.set("hbase.security.authentication", "kerberos");
110    Assert.assertTrue(User.isHBaseSecurityEnabled(conf));
111
112    conf.set("hbase.security.authentication", "KERBEROS");
113    Assert.assertTrue(User.isHBaseSecurityEnabled(conf));
114
115    conf.set("hbase.security.authentication", "KERBeros");
116    Assert.assertTrue(User.isHBaseSecurityEnabled(conf));
117  }
118
119  @Test
120  public void testGetConfigOfShortcircuitRead() throws Exception {
121    Configuration conf = HBaseConfiguration.create();
122    Configuration.addDefaultResource("hdfs-scr-disabled.xml");
123    assertEquals("hdfs-scr-disabled.xml",
124            conf.getPropertySources("dfs.client.read.shortcircuit")[0]);
125    assertEquals("false", conf.get("dfs.client.read.shortcircuit"));
126    assertNull(conf.get("dfs.domain.socket.path"));
127    Configuration.addDefaultResource("hdfs-scr-enabled.xml");
128    assertEquals("hdfs-scr-enabled.xml",
129            conf.getPropertySources("dfs.client.read.shortcircuit")[0]);
130    assertEquals("hdfs-scr-enabled.xml",
131            conf.getPropertySources("dfs.domain.socket.path")[0]);
132    assertEquals("true", conf.get("dfs.client.read.shortcircuit"));
133    assertEquals("/var/lib/hadoop-hdfs/dn_socket", conf.get("dfs.domain.socket.path"));
134  }
135
136  @Test
137  public void testDeprecatedConfigurations() {
138    // Configuration.addDeprecations before create Configuration object
139    Configuration.addDeprecations(new Configuration.DeprecationDelta[]{
140      new Configuration.DeprecationDelta("hbase.deprecated.conf", "hbase.new.conf"),
141      new Configuration.DeprecationDelta("hbase.deprecated.conf2", "hbase.new.conf2")
142    });
143    Configuration conf = HBaseConfiguration.create();
144    conf.addResource("hbase-deprecated-conf.xml");
145    assertEquals("1000", conf.get("hbase.new.conf"));
146    assertEquals("1000", conf.get("hbase.new.conf2"));
147  }
148
149  private static class ReflectiveCredentialProviderClient {
150    public static final String HADOOP_CRED_PROVIDER_FACTORY_CLASS_NAME =
151        "org.apache.hadoop.security.alias.JavaKeyStoreProvider$Factory";
152    public static final String
153      HADOOP_CRED_PROVIDER_FACTORY_GET_PROVIDERS_METHOD_NAME = "getProviders";
154
155    public static final String HADOOP_CRED_PROVIDER_CLASS_NAME =
156        "org.apache.hadoop.security.alias.CredentialProvider";
157    public static final String
158        HADOOP_CRED_PROVIDER_GET_CREDENTIAL_ENTRY_METHOD_NAME =
159        "getCredentialEntry";
160    public static final String
161        HADOOP_CRED_PROVIDER_GET_ALIASES_METHOD_NAME = "getAliases";
162    public static final String
163        HADOOP_CRED_PROVIDER_CREATE_CREDENTIAL_ENTRY_METHOD_NAME =
164        "createCredentialEntry";
165    public static final String HADOOP_CRED_PROVIDER_FLUSH_METHOD_NAME = "flush";
166
167    public static final String HADOOP_CRED_ENTRY_CLASS_NAME =
168        "org.apache.hadoop.security.alias.CredentialProvider$CredentialEntry";
169    public static final String HADOOP_CRED_ENTRY_GET_CREDENTIAL_METHOD_NAME =
170        "getCredential";
171
172    public static final String CREDENTIAL_PROVIDER_PATH =
173        "hadoop.security.credential.provider.path";
174
175    private static Object hadoopCredProviderFactory = null;
176    private static Method getProvidersMethod = null;
177    private static Method getCredentialEntryMethod = null;
178    private static Method getCredentialMethod = null;
179    private static Method createCredentialEntryMethod = null;
180    private static Method flushMethod = null;
181    private static Boolean hadoopClassesAvailable = null;
182
183    /**
184     * Determine if we can load the necessary CredentialProvider classes. Only
185     * loaded the first time, so subsequent invocations of this method should
186     * return fast.
187     *
188     * @return True if the CredentialProvider classes/methods are available,
189     *         false otherwise.
190     */
191    private boolean isHadoopCredentialProviderAvailable() {
192      if (null != hadoopClassesAvailable) {
193        // Make sure everything is initialized as expected
194        if (hadoopClassesAvailable && null != getProvidersMethod
195            && null != hadoopCredProviderFactory
196            && null != getCredentialEntryMethod && null != getCredentialMethod) {
197          return true;
198        } else {
199          // Otherwise we failed to load it
200          return false;
201        }
202      }
203
204      hadoopClassesAvailable = false;
205
206      // Load Hadoop CredentialProviderFactory
207      Class<?> hadoopCredProviderFactoryClz;
208      try {
209        hadoopCredProviderFactoryClz = Class
210            .forName(HADOOP_CRED_PROVIDER_FACTORY_CLASS_NAME);
211      } catch (ClassNotFoundException e) {
212        return false;
213      }
214      // Instantiate Hadoop CredentialProviderFactory
215      try {
216        hadoopCredProviderFactory =
217          hadoopCredProviderFactoryClz.getDeclaredConstructor().newInstance();
218      } catch (Exception e) {
219        return false;
220      }
221
222      try {
223        getProvidersMethod = loadMethod(hadoopCredProviderFactoryClz,
224            HADOOP_CRED_PROVIDER_FACTORY_GET_PROVIDERS_METHOD_NAME,
225            Configuration.class);
226        // Load Hadoop CredentialProvider
227        Class<?> hadoopCredProviderClz;
228        hadoopCredProviderClz = Class.forName(HADOOP_CRED_PROVIDER_CLASS_NAME);
229        getCredentialEntryMethod = loadMethod(hadoopCredProviderClz,
230            HADOOP_CRED_PROVIDER_GET_CREDENTIAL_ENTRY_METHOD_NAME, String.class);
231
232        Method getAliasesMethod =
233          loadMethod(hadoopCredProviderClz, HADOOP_CRED_PROVIDER_GET_ALIASES_METHOD_NAME);
234
235        createCredentialEntryMethod = loadMethod(hadoopCredProviderClz,
236            HADOOP_CRED_PROVIDER_CREATE_CREDENTIAL_ENTRY_METHOD_NAME,
237            String.class, char[].class);
238
239        flushMethod = loadMethod(hadoopCredProviderClz,
240            HADOOP_CRED_PROVIDER_FLUSH_METHOD_NAME);
241
242        // Load Hadoop CredentialEntry
243        Class<?> hadoopCredentialEntryClz;
244        try {
245          hadoopCredentialEntryClz = Class
246              .forName(HADOOP_CRED_ENTRY_CLASS_NAME);
247        } catch (ClassNotFoundException e) {
248          LOG.error("Failed to load class:" + e);
249          return false;
250        }
251
252        getCredentialMethod = loadMethod(hadoopCredentialEntryClz,
253            HADOOP_CRED_ENTRY_GET_CREDENTIAL_METHOD_NAME);
254      } catch (Exception e1) {
255        return false;
256      }
257
258      hadoopClassesAvailable = true;
259      LOG.info("Credential provider classes have been" +
260          " loaded and initialized successfully through reflection.");
261      return true;
262    }
263
264    private Method loadMethod(Class<?> clz, String name, Class<?>... classes)
265        throws Exception {
266      Method method;
267      try {
268        method = clz.getMethod(name, classes);
269      } catch (SecurityException e) {
270        fail("security exception caught for: " + name + " in " + clz.getCanonicalName());
271        throw e;
272      } catch (NoSuchMethodException e) {
273        LOG.error("Failed to load the " + name + ": " + e);
274        fail("no such method: " + name + " in " + clz.getCanonicalName());
275        throw e;
276      }
277      return method;
278    }
279
280    /**
281     * Wrapper to fetch the configured {@code List<CredentialProvider>}s.
282     *
283     * @param conf
284     *    Configuration with GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS defined
285     * @return List of CredentialProviders, or null if they could not be loaded
286     */
287    @SuppressWarnings("unchecked")
288    protected  List<Object> getCredentialProviders(Configuration conf) {
289      // Call CredentialProviderFactory.getProviders(Configuration)
290      Object providersObj;
291      try {
292        providersObj = getProvidersMethod.invoke(hadoopCredProviderFactory,
293            conf);
294      } catch (IllegalArgumentException | InvocationTargetException | IllegalAccessException e) {
295        LOG.error("Failed to invoke: " + getProvidersMethod.getName() +
296            ": " + e);
297        return null;
298      }
299
300      // Cast the Object to List<Object> (actually List<CredentialProvider>)
301      try {
302        return (List<Object>) providersObj;
303      } catch (ClassCastException e) {
304        return null;
305      }
306    }
307
308    /**
309     * Create a CredentialEntry using the configured Providers.
310     * If multiple CredentialProviders are configured, the first will be used.
311     *
312     * @param conf
313     *          Configuration for the CredentialProvider
314     * @param name
315     *          CredentialEntry name (alias)
316     * @param credential
317     *          The credential
318     */
319    public  void createEntry(Configuration conf, String name, char[] credential)
320        throws Exception {
321      if (!isHadoopCredentialProviderAvailable()) {
322        return;
323      }
324
325      List<Object> providers = getCredentialProviders(conf);
326      if (null == providers) {
327        throw new IOException("Could not fetch any CredentialProviders, " +
328            "is the implementation available?");
329      }
330
331      Object provider = providers.get(0);
332      createEntryInProvider(provider, name, credential);
333    }
334
335    /**
336     * Create a CredentialEntry with the give name and credential in the
337     * credentialProvider. The credentialProvider argument must be an instance
338     * of Hadoop
339     * CredentialProvider.
340     *
341     * @param credentialProvider
342     *          Instance of CredentialProvider
343     * @param name
344     *          CredentialEntry name (alias)
345     * @param credential
346     *          The credential to store
347     */
348    private void createEntryInProvider(Object credentialProvider,
349        String name, char[] credential) throws Exception {
350      if (!isHadoopCredentialProviderAvailable()) {
351        return;
352      }
353
354      try {
355        createCredentialEntryMethod.invoke(credentialProvider, name, credential);
356      } catch (IllegalArgumentException | InvocationTargetException | IllegalAccessException e) {
357        return;
358      }
359
360      flushMethod.invoke(credentialProvider);
361    }
362  }
363}