From 65a9340fc22fcc3c1d2c64b3976994aa1c419b8f Mon Sep 17 00:00:00 2001 From: Abraham Elmahrek Date: Wed, 8 Oct 2014 13:59:47 -0700 Subject: [PATCH] SQOOP-1471: Use Hadoop CredentialProvider API to encyrpt passwords at rest (Venkat Ranganathan via Abraham Elmahrek) --- src/docs/user/connecting.txt | 47 ++++++ src/java/org/apache/sqoop/SqoopOptions.java | 8 + .../org/apache/sqoop/tool/BaseSqoopTool.java | 41 ++++- .../password/CredentialProviderHelper.java | 151 ++++++++++++++++++ .../CredentialProviderPasswordLoader.java | 59 +++++++ .../util/password/FilePasswordLoader.java | 6 +- .../TestPassingSecurePassword.java | 105 +++++++++++- 7 files changed, 400 insertions(+), 17 deletions(-) create mode 100644 src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java create mode 100644 src/java/org/apache/sqoop/util/password/CredentialProviderPasswordLoader.java diff --git a/src/docs/user/connecting.txt b/src/docs/user/connecting.txt index 6a282546..fd812c0e 100644 --- a/src/docs/user/connecting.txt +++ b/src/docs/user/connecting.txt @@ -63,6 +63,53 @@ $ sqoop import --connect jdbc:mysql://database.example.com/employees \ Another way of supplying passwords is using the +-P+ argument which will read a password from a console prompt. +.Protecting password from preying eyes +Hadoop 2.6.0 provides an API to separate password storage from applications. +This API is called the credential provided API and there is a new ++credential+ command line tool to manage passwords and their aliases. +The passwords are stored with their aliases in a keystore that is password +protected. The keystore password can be the provided to a password prompt +on the command line, via an environment variable or defaulted to a software +defined constant. Please check the Hadoop documentation on the usage +of this facility. + +Once the password is stored using the Credential Provider facility and +the Hadoop configuration has been suitably updated, all applications can +optionally use the alias in place of the actual password and at runtime +resolve the alias for the password to use. + +Since the keystore or similar technology used for storing the credential +provider is shared across components, passwords for various applications, +various database and other passwords can be securely stored in them and only +the alias needs to be exposed in configuration files, protecting the password +from being visible. + +Sqoop has been enhanced to allow usage of this funcionality if it is +available in the underlying Hadoop version being used. One new option +has been introduced to provide the alias on the command line instead of the +actual password (--password-alias). The argument value this option is +the alias on the storage associated with the actual password. +Example usage is as follows: + +---- +$ sqoop import --connect jdbc:mysql://database.example.com/employees \ + --username dbuser --password-alias mydb.password.alias +---- + +Similarly, if the command line option is not preferred, the alias can be saved +in the file provided with --password-file option. Along with this, the +Sqoop configuration parameter org.apache.sqoop.credentials.loader.class +should be set to the classname that provides the alias resolution: ++org.apache.sqoop.util.password.CredentialProviderPasswordLoader+ + +Example usage is as follows (assuming .password.alias has the alias for +the real password) : + +---- +$ sqoop import --connect jdbc:mysql://database.example.com/employees \ + --username dbuser --password-file ${user.home}/.password-alias +---- + .Non-secure way of passing password WARNING: The +\--password+ parameter is insecure, as other users may diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index d16ccb39..0070d0bd 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -119,6 +119,7 @@ public String toString() { // This represents path to a file on ${user.home} containing the password // with 400 permissions so its only readable by user executing the tool @StoredAsProperty("db.password.file") private String passwordFilePath; + @StoredAsProperty("db.password.alias") private String passwordAlias; @StoredAsProperty("null.string") private String nullStringValue; @StoredAsProperty("input.null.string") private String inNullStringValue; @@ -1183,6 +1184,13 @@ public void setPasswordFilePath(String passwdFilePath) { this.passwordFilePath = passwdFilePath; } + public String getPasswordAlias() { + return passwordAlias; + } + + public void setPasswordAlias(String alias) { + this.passwordAlias = alias; + } protected void parseColumnMapping(String mapping, Properties output) { output.clear(); diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 498ad79f..b41ee2d2 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -35,6 +35,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.sqoop.util.CredentialsUtil; import org.apache.sqoop.util.LoggingUtils; +import org.apache.sqoop.util.password.CredentialProviderHelper; import com.cloudera.sqoop.ConnFactory; import com.cloudera.sqoop.Sqoop; @@ -73,6 +74,7 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { public static final String PASSWORD_ARG = "password"; public static final String PASSWORD_PROMPT_ARG = "P"; public static final String PASSWORD_PATH_ARG = "password-file"; + public static final String PASSWORD_ALIAS_ARG = "password-alias"; public static final String DIRECT_ARG = "direct"; public static final String BATCH_ARG = "batch"; public static final String TABLE_ARG = "table"; @@ -426,7 +428,10 @@ protected RelatedOptions getCommonOptions() { commonOpts.addOption(OptionBuilder .withDescription("Read password from console") .create(PASSWORD_PROMPT_ARG)); - + commonOpts.addOption(OptionBuilder.withArgName(PASSWORD_ALIAS_ARG) + .hasArg().withDescription("Credential provider password alias") + .withLongOpt(PASSWORD_ALIAS_ARG) + .create()); commonOpts.addOption(OptionBuilder.withArgName("dir") .hasArg().withDescription("Override $HADOOP_MAPRED_HOME_ARG") .withLongOpt(HADOOP_MAPRED_HOME_ARG) @@ -1017,9 +1022,10 @@ private void applyCredentialsOptions(CommandLine in, SqoopOptions out) } if (in.hasOption(PASSWORD_PATH_ARG)) { - if (in.hasOption(PASSWORD_ARG) || in.hasOption(PASSWORD_PROMPT_ARG)) { - throw new InvalidOptionsException("Either password or path to a " - + "password file must be specified but not both."); + if (in.hasOption(PASSWORD_ARG) || in.hasOption(PASSWORD_PROMPT_ARG) + || in.hasOption(PASSWORD_ALIAS_ARG)) { + throw new InvalidOptionsException("Only one of password, password " + + "alias or path to a password file must be specified."); } try { @@ -1029,10 +1035,31 @@ private void applyCredentialsOptions(CommandLine in, SqoopOptions out) // And allow the PasswordLoader to clean up any sensitive properties CredentialsUtil.cleanUpSensitiveProperties(out.getConf()); } catch (IOException ex) { - LOG.warn("Failed to load connection parameter file", ex); + LOG.warn("Failed to load password file", ex); + throw (InvalidOptionsException) + new InvalidOptionsException("Error while loading password file: " + + ex.getMessage()).initCause(ex); + } + } + if (in.hasOption(PASSWORD_ALIAS_ARG)) { + if (in.hasOption(PASSWORD_ARG) || in.hasOption(PASSWORD_PROMPT_ARG) + || in.hasOption(PASSWORD_PATH_ARG)) { + throw new InvalidOptionsException("Only one of password, password " + + "alias or path to a password file must be specified."); + } + out.setPasswordAlias(in.getOptionValue(PASSWORD_ALIAS_ARG)); + if (!CredentialProviderHelper.isProviderAvailable()) { throw new InvalidOptionsException( - "Error while loading connection parameter file: " - + ex.getMessage()); + "CredentialProvider facility not available in the hadoop " + + " environment used"); + } + try { + out.setPassword(CredentialProviderHelper + .resolveAlias(out.getConf(), in.getOptionValue(PASSWORD_ALIAS_ARG))); + } catch (IOException ioe) { + throw (InvalidOptionsException) + new InvalidOptionsException("Unable to process alias") + .initCause(ioe); } } } diff --git a/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java b/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java new file mode 100644 index 00000000..1d6481a0 --- /dev/null +++ b/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.util.password; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; + +/** + * Helper class for the Hadoop credential provider functionality. + * Reflrection to used to avoid directly referencing the classes and methods + * so that version dependency is not introduced as the Hadoop credential + * provider is only introduced in 2.6.0 and later + */ +public class CredentialProviderHelper { + public static final Log LOG = + LogFactory.getLog(CredentialProviderHelper.class.getName()); + + private static Class clsCredProvider; + private static Class clsCredProviderFactory; + private static Method methGetPassword; + private static Method methGetProviders; + private static Method methCreateCredEntry; + private static Method methFlush; + + static { + try { + LOG.debug("Reflecting credential provider classes and methods"); + clsCredProvider = Class + .forName("org.apache.hadoop.security.alias.CredentialProvider"); + LOG + .debug("Found org.apache.hadoop.security.alias.CredentialProvider"); + clsCredProviderFactory = Class.forName( + "org.apache.hadoop.security.alias.CredentialProviderFactory"); + LOG + .debug("Found org.apache.hadoop.security.alias.CredentialProviderFactory"); + + methCreateCredEntry = clsCredProvider.getMethod("createCredentialEntry", + new Class[] { String.class, char[].class }); + LOG + .debug("Found CredentialProvider#createCredentialEntry"); + + methFlush = clsCredProvider.getMethod("flush", + new Class[] {}); + LOG + .debug("Found CredentialProvider#flush"); + + methGetPassword = Configuration.class.getMethod("getPassword", + new Class[] { String.class }); + LOG + .debug("Found Configuration#getPassword"); + + methGetProviders = clsCredProviderFactory.getMethod("getProviders", + new Class[] { Configuration.class }); + LOG + .debug("Found CredentialProviderFactory#getProviders"); + } catch (ClassNotFoundException cnfe) { + LOG.debug("Ignoring exception", cnfe); + } catch (NoSuchMethodException nsme) { + LOG.debug("Ignoring exception", nsme); + } + } + // Should track what is specified in JavaKeyStoreProvider class. + public static final String SCHEME_NAME = "jceks"; + // Should track what is in CredentialProvider class. + public static final String CREDENTIAL_PROVIDER_PATH = + "hadoop.security.credential.provider.path"; + + public static boolean isProviderAvailable() { + + if (clsCredProvider == null + || clsCredProviderFactory == null + || methCreateCredEntry == null + || methGetPassword == null + || methFlush == null) { + return false; + } + return true; + } + + public static String resolveAlias(Configuration conf, String alias) + throws IOException { + LOG.debug("Resolving alias with credential provider path set to " + + conf.get(CREDENTIAL_PROVIDER_PATH)); + try { + char[] cred = (char[]) + methGetPassword.invoke(conf, new Object[] { alias }); + if (cred == null) { + throw new IOException("The provided alias cannot be resolved"); + } + String pass = new String(cred); + return pass; + } catch (InvocationTargetException ite) { + throw new RuntimeException("Error resolving password " + + " from the credential providers ", ite.getTargetException()); + } catch (IllegalAccessException iae) { + throw new RuntimeException("Error invoking the credential provider method", + iae); + } + } + /** + * Test utility to create an entry + */ + public static void createCredentialEntry(Configuration conf, + + String alias, String credential) throws IOException { + + if (!isProviderAvailable()) { + throw new RuntimeException("CredentialProvider facility not available " + + "in the hadoop environment"); + } + + + try { + List result = (List) + methGetProviders.invoke(null, new Object[] { conf }); + Object provider = result.get(0); + LOG.debug("Using credential provider " + provider); + + methCreateCredEntry.invoke(provider, new Object[] { + alias, credential.toCharArray() }); + methFlush.invoke(provider, new Object[] {}); + } catch (InvocationTargetException ite) { + throw new RuntimeException("Error creating credential entry " + + " using the credentail provider", ite.getTargetException()); + } catch (IllegalAccessException iae) { + throw new RuntimeException("Error accessing the credential create method", + iae); + } + } +} diff --git a/src/java/org/apache/sqoop/util/password/CredentialProviderPasswordLoader.java b/src/java/org/apache/sqoop/util/password/CredentialProviderPasswordLoader.java new file mode 100644 index 00000000..501f130b --- /dev/null +++ b/src/java/org/apache/sqoop/util/password/CredentialProviderPasswordLoader.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.util.password; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +/** + * A password loader that loads an credential provider alias. + * The alias is resolved using the Hadoop credential provider facilitity + * if available. + */ +public class CredentialProviderPasswordLoader extends FilePasswordLoader { + public static final Log LOG = + LogFactory.getLog(CredentialProviderPasswordLoader.class.getName()); + + /** + * If credential provider is available (made available as part of 2.6.0 and + * 3.0, then use the credential provider to get the password. Else throw an + * exception saying this provider is not available. + */ + @Override + public String loadPassword(String p, Configuration configuration) + throws IOException { + if (!CredentialProviderHelper.isProviderAvailable()) { + throw new IOException("CredentialProvider facility not available " + + "in the hadoop environment used"); + } + LOG.debug("Fetching alias from the specified path: " + p); + Path path = new Path(p); + FileSystem fs = path.getFileSystem(configuration); + + // Not closing FileSystem object because of SQOOP-1226 + verifyPath(fs, path); + String alias = new String(readBytes(fs, path)); + String pass = CredentialProviderHelper.resolveAlias(configuration, alias); + return pass; + } +} diff --git a/src/java/org/apache/sqoop/util/password/FilePasswordLoader.java b/src/java/org/apache/sqoop/util/password/FilePasswordLoader.java index 4a288bfc..7f0c8b40 100644 --- a/src/java/org/apache/sqoop/util/password/FilePasswordLoader.java +++ b/src/java/org/apache/sqoop/util/password/FilePasswordLoader.java @@ -48,11 +48,13 @@ public class FilePasswordLoader extends PasswordLoader { */ protected void verifyPath(FileSystem fs, Path path) throws IOException { if (!fs.exists(path)) { - throw new IOException("The password file does not exist! " + path); + throw new IOException("The provided password file " + path + + " does not exist!"); } if (!fs.isFile(path)) { - throw new IOException("The password file cannot be a directory! " + path); + throw new IOException("The provided password file " + path + + " is a directory!"); } } diff --git a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java index bbf82f4c..5b170b66 100644 --- a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java +++ b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java @@ -21,6 +21,7 @@ import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.testutil.BaseSqoopTestCase; import com.cloudera.sqoop.testutil.CommonArgs; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -29,13 +30,17 @@ import org.apache.sqoop.mapreduce.db.DBConfiguration; import org.apache.sqoop.tool.BaseSqoopTool; import org.apache.sqoop.tool.ImportTool; +import org.apache.sqoop.util.password.CredentialProviderHelper; +import org.apache.sqoop.util.password.CredentialProviderPasswordLoader; import org.apache.sqoop.util.password.CryptoFileLoader; +import org.apache.sqoop.util.password.PasswordLoader; import javax.crypto.Cipher; import javax.crypto.SecretKey; import javax.crypto.SecretKeyFactory; import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.SecretKeySpec; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -93,9 +98,10 @@ public void testPasswordFileDoesNotExist() throws Exception { SqoopOptions opts = getSqoopOptions(conf); ImportTool importTool = new ImportTool(); importTool.parseArguments(argv, conf, opts, true); - fail("The password file does not exist! "); + fail("The password file does not exist!"); } catch (Exception e) { - assertTrue(e.getMessage().contains("The password file does not exist!")); + assertTrue(e.getMessage().matches(".*The provided password file " + + ".* does not exist!")); } } @@ -110,10 +116,10 @@ public void testPasswordFileIsADirectory() throws Exception { SqoopOptions opts = getSqoopOptions(conf); ImportTool importTool = new ImportTool(); importTool.parseArguments(argv, conf, opts, true); - fail("The password file cannot be a directory! "); + fail("The password file cannot be a directory!"); } catch (Exception e) { - assertTrue(e.getMessage().contains("The password file cannot " - + "be a directory!")); + assertTrue(e.getMessage().matches(".*The provided password file .*" + + " is a directory!")); } } @@ -137,10 +143,11 @@ public void testBothPasswordOptions() throws Exception { SqoopOptions out = importTool.parseArguments(argv, conf, in, true); assertNotNull(out.getPassword()); importTool.validateOptions(out); - fail("Either password or passwordPath must be specified but not both."); + fail("Only one of password, password " + + "alias or path to a password file must be specified."); } catch (Exception e) { - assertTrue(e.getMessage().contains("Either password or path to a " - + "password file must be specified but not both")); + assertTrue(e.getMessage().contains("Only one of password, password " + + "alias or path to a password file must be specified.")); } } @@ -351,6 +358,88 @@ public void testCryptoFileLoader() throws Exception { } } + public void testCredentialProviderLoader() throws Exception { + CredentialProviderPasswordLoader pl = + new CredentialProviderPasswordLoader(); + + if (!CredentialProviderHelper.isProviderAvailable()) { + LOG.info("CredentialProvider facility not available " + + "in the hadoop environment used"); + } else { + String alias = "super.secret.alias"; + String pw = "super.secret.password"; + + String passwordFilePath = TEMP_BASE_DIR + ".pwd"; + String jksFile = "creds.jks"; + createTempFile(passwordFilePath); + writeToFile(passwordFilePath, alias.getBytes()); + File credDir = new File("."); + + Configuration conf = getConf(); + String ourUrl = CredentialProviderHelper.SCHEME_NAME + + "://file/" + credDir.getAbsolutePath() + "/" + jksFile; + File file = new File(credDir, jksFile); + file.delete(); + conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, + ourUrl); + CredentialProviderHelper.createCredentialEntry(conf, alias, pw); + + conf.set("org.apache.sqoop.credentials.loader.class", + CredentialProviderPasswordLoader.class.getCanonicalName()); + + ArrayList extraArgs = new ArrayList(); + extraArgs.add("--username"); + extraArgs.add("username"); + extraArgs.add("--password-file"); + extraArgs.add(passwordFilePath); + String[] commonArgs = getCommonArgs(false, extraArgs); + + SqoopOptions in = getSqoopOptions(conf); + ImportTool importTool = new ImportTool(); + + SqoopOptions out = importTool.parseArguments(commonArgs, conf, in, true); + assertEquals(pw, pl.loadPassword(passwordFilePath, conf)); + assertEquals(pw, out.getPassword()); + } + } + + public void testPasswordAliasOption() throws Exception { + CredentialProviderPasswordLoader pl = + new CredentialProviderPasswordLoader(); + + if (!CredentialProviderHelper.isProviderAvailable()) { + LOG.info("CredentialProvider facility not available " + + "in the hadoop environment used"); + } else { + String alias = "super.secret.alias"; + String pw = "super.secret.password"; + String jksFile = "creds.jks"; + File credDir = new File("."); + + Configuration conf = getConf(); + String ourUrl = CredentialProviderHelper.SCHEME_NAME + + "://file/" + credDir.getAbsolutePath() + "/" + jksFile; + File file = new File(credDir, jksFile); + file.delete(); + conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, + ourUrl); + CredentialProviderHelper.createCredentialEntry(conf, alias, pw); + + ArrayList extraArgs = new ArrayList(); + extraArgs.add("--username"); + extraArgs.add("username"); + extraArgs.add("--password-alias"); + extraArgs.add(alias); + String[] commonArgs = getCommonArgs(false, extraArgs); + + SqoopOptions in = getSqoopOptions(conf); + ImportTool importTool = new ImportTool(); + + SqoopOptions out = importTool.parseArguments(commonArgs, conf, in, true); + assertEquals(pw, out.getPassword()); + } + } + public void executeCipherTest(String password, String passphrase, String cipher, int keySize) throws Exception { LOG.info("Using cipher: " + cipher + " with keySize " + keySize + " and passphrase " + passphrase ); String passwordFilePath = TEMP_BASE_DIR + ".pwd";