SQOOP-1471: Use Hadoop CredentialProvider API to encyrpt passwords at rest

(Venkat Ranganathan via Abraham Elmahrek)

src/docs/user/connecting.txt

@@ -63,6 +63,53 @@ $ sqoop import --connect jdbc:mysql://database.example.com/employees \
 Another way of supplying passwords is using the +-P+ argument which will
 read a password from a console prompt.
 
+.Protecting password from preying eyes
+Hadoop 2.6.0 provides an API to separate password storage from applications.
+This API is called the credential provided API and there is a new
++credential+  command line tool to manage passwords and their aliases.
+The passwords are stored with their aliases in a keystore that is password
+protected.   The keystore password can be the provided to a password prompt
+on the command line, via an environment variable or defaulted to a software
+defined constant.   Please check the Hadoop documentation on the usage
+of this facility.
+
+Once the password is stored using the Credential Provider facility and
+the Hadoop configuration has been suitably updated, all applications can
+optionally use the alias in place of the actual password and at runtime
+resolve the alias for the password to use.
+
+Since the keystore or similar technology used for storing the credential
+provider is shared across components, passwords for various applications,
+various database and other passwords can be securely stored in them and only
+the alias needs to be exposed in configuration files, protecting the password
+from being visible.
+
+Sqoop has been enhanced to allow usage of this funcionality if it is
+available in the underlying Hadoop version being used.   One new option
+has been introduced to provide the alias on the command line instead of the
+actual password (--password-alias).  The argument value this option is
+the alias on the storage associated with the actual password.
+Example usage is as follows:
+
+----
+$ sqoop import --connect jdbc:mysql://database.example.com/employees \
+    --username dbuser --password-alias mydb.password.alias
+----
+
+Similarly, if the command line option is not preferred, the alias can be saved
+in the file provided with --password-file option.  Along with this, the
+Sqoop configuration parameter org.apache.sqoop.credentials.loader.class
+should be set to the classname that provides the alias resolution:
++org.apache.sqoop.util.password.CredentialProviderPasswordLoader+
+
+Example usage is as follows (assuming .password.alias has the alias for
+the real password) :
+
+----
+$ sqoop import --connect jdbc:mysql://database.example.com/employees \
+    --username dbuser --password-file ${user.home}/.password-alias
+----
+
 .Non-secure way of passing password
 
 WARNING: The +\--password+ parameter is insecure, as other users may

src/java/org/apache/sqoop/SqoopOptions.java

@@ -119,6 +119,7 @@ public String toString() {
   // This represents path to a file on ${user.home} containing the password
   // with 400 permissions so its only readable by user executing the tool
   @StoredAsProperty("db.password.file") private String passwordFilePath;
+  @StoredAsProperty("db.password.alias") private String passwordAlias;
 
   @StoredAsProperty("null.string") private String nullStringValue;
   @StoredAsProperty("input.null.string") private String inNullStringValue;
@@ -1183,6 +1184,13 @@ public void setPasswordFilePath(String passwdFilePath) {
     this.passwordFilePath = passwdFilePath;
   }
 
+  public String getPasswordAlias() {
+    return passwordAlias;
+  }
+
+  public void setPasswordAlias(String alias) {
+    this.passwordAlias = alias;
+  }
   protected void parseColumnMapping(String mapping,
           Properties output) {
     output.clear();

src/java/org/apache/sqoop/tool/BaseSqoopTool.java

@@ -35,6 +35,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.sqoop.util.CredentialsUtil;
 import org.apache.sqoop.util.LoggingUtils;
+import org.apache.sqoop.util.password.CredentialProviderHelper;
 
 import com.cloudera.sqoop.ConnFactory;
 import com.cloudera.sqoop.Sqoop;
@@ -73,6 +74,7 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
   public static final String PASSWORD_ARG = "password";
   public static final String PASSWORD_PROMPT_ARG = "P";
   public static final String PASSWORD_PATH_ARG = "password-file";
+  public static final String PASSWORD_ALIAS_ARG = "password-alias";
   public static final String DIRECT_ARG = "direct";
   public static final String BATCH_ARG = "batch";
   public static final String TABLE_ARG = "table";
@@ -426,7 +428,10 @@ protected RelatedOptions getCommonOptions() {
     commonOpts.addOption(OptionBuilder
         .withDescription("Read password from console")
         .create(PASSWORD_PROMPT_ARG));
-
+    commonOpts.addOption(OptionBuilder.withArgName(PASSWORD_ALIAS_ARG)
+      .hasArg().withDescription("Credential provider password alias")
+      .withLongOpt(PASSWORD_ALIAS_ARG)
+      .create());
     commonOpts.addOption(OptionBuilder.withArgName("dir")
         .hasArg().withDescription("Override $HADOOP_MAPRED_HOME_ARG")
         .withLongOpt(HADOOP_MAPRED_HOME_ARG)
@@ -1017,9 +1022,10 @@ private void applyCredentialsOptions(CommandLine in, SqoopOptions out)
     }
 
     if (in.hasOption(PASSWORD_PATH_ARG)) {
-      if (in.hasOption(PASSWORD_ARG) || in.hasOption(PASSWORD_PROMPT_ARG)) {
+      if (in.hasOption(PASSWORD_ARG) || in.hasOption(PASSWORD_PROMPT_ARG)
-        throw new InvalidOptionsException("Either password or path to a "
+          || in.hasOption(PASSWORD_ALIAS_ARG)) {
-          + "password file must be specified but not both.");
+        throw new InvalidOptionsException("Only one of password, password "
+          + "alias or path to a password file must be specified.");
       }
 
       try {
@@ -1029,10 +1035,31 @@ private void applyCredentialsOptions(CommandLine in, SqoopOptions out)
         // And allow the PasswordLoader to clean up any sensitive properties
         CredentialsUtil.cleanUpSensitiveProperties(out.getConf());
       } catch (IOException ex) {
-        LOG.warn("Failed to load connection parameter file", ex);
+        LOG.warn("Failed to load password file", ex);
+        throw (InvalidOptionsException)
+          new InvalidOptionsException("Error while loading password file: "
+            + ex.getMessage()).initCause(ex);
+      }
+    }
+    if (in.hasOption(PASSWORD_ALIAS_ARG)) {
+      if (in.hasOption(PASSWORD_ARG) || in.hasOption(PASSWORD_PROMPT_ARG)
+          || in.hasOption(PASSWORD_PATH_ARG)) {
+        throw new InvalidOptionsException("Only one of password, password "
+          + "alias or path to a password file must be specified.");
+      }
+      out.setPasswordAlias(in.getOptionValue(PASSWORD_ALIAS_ARG));
+      if (!CredentialProviderHelper.isProviderAvailable()) {
         throw new InvalidOptionsException(
-          "Error while loading connection parameter file: "
+          "CredentialProvider facility not available in the hadoop "
-            + ex.getMessage());
+          + " environment used");
+      }
+      try {
+        out.setPassword(CredentialProviderHelper
+          .resolveAlias(out.getConf(), in.getOptionValue(PASSWORD_ALIAS_ARG)));
+      } catch (IOException ioe) {
+        throw (InvalidOptionsException)
+          new InvalidOptionsException("Unable to process alias")
+            .initCause(ioe);
       }
     }
   }

src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java

@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.util.password;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Helper class for the Hadoop credential provider functionality.
+ * Reflrection to used to avoid directly referencing the classes and methods
+ * so that version dependency is not introduced as the Hadoop credential
+ * provider is only introduced in 2.6.0 and later
+ */
+public class CredentialProviderHelper {
+  public static final Log LOG =
+    LogFactory.getLog(CredentialProviderHelper.class.getName());
+
+  private static Class<?> clsCredProvider;
+  private static Class<?> clsCredProviderFactory;
+  private static Method methGetPassword;
+  private static Method methGetProviders;
+  private static Method methCreateCredEntry;
+  private static Method methFlush;
+
+  static {
+    try {
+      LOG.debug("Reflecting credential provider classes and methods");
+      clsCredProvider = Class
+        .forName("org.apache.hadoop.security.alias.CredentialProvider");
+      LOG
+        .debug("Found org.apache.hadoop.security.alias.CredentialProvider");
+      clsCredProviderFactory = Class.forName(
+        "org.apache.hadoop.security.alias.CredentialProviderFactory");
+      LOG
+        .debug("Found org.apache.hadoop.security.alias.CredentialProviderFactory");
+
+      methCreateCredEntry = clsCredProvider.getMethod("createCredentialEntry",
+        new Class[] { String.class, char[].class });
+      LOG
+        .debug("Found CredentialProvider#createCredentialEntry");
+
+      methFlush = clsCredProvider.getMethod("flush",
+        new Class[] {});
+      LOG
+        .debug("Found CredentialProvider#flush");
+
+      methGetPassword = Configuration.class.getMethod("getPassword",
+        new Class[] { String.class });
+      LOG
+        .debug("Found Configuration#getPassword");
+
+      methGetProviders = clsCredProviderFactory.getMethod("getProviders",
+        new Class[] { Configuration.class });
+      LOG
+        .debug("Found CredentialProviderFactory#getProviders");
+    } catch (ClassNotFoundException cnfe) {
+      LOG.debug("Ignoring exception", cnfe);
+    } catch (NoSuchMethodException nsme) {
+      LOG.debug("Ignoring exception", nsme);
+    }
+  }
+  // Should track what is specified in JavaKeyStoreProvider class.
+  public static final String SCHEME_NAME = "jceks";
+  // Should track what is in CredentialProvider class.
+  public static final String CREDENTIAL_PROVIDER_PATH =
+    "hadoop.security.credential.provider.path";
+
+  public static boolean isProviderAvailable() {
+
+    if (clsCredProvider == null
+      || clsCredProviderFactory == null
+      || methCreateCredEntry == null
+      || methGetPassword == null
+      || methFlush == null) {
+      return false;
+    }
+    return true;
+  }
+
+  public static String resolveAlias(Configuration conf, String alias)
+    throws IOException {
+    LOG.debug("Resolving alias with credential provider path set to "
+      + conf.get(CREDENTIAL_PROVIDER_PATH));
+    try {
+      char[] cred = (char[])
+        methGetPassword.invoke(conf, new Object[] { alias });
+      if (cred == null) {
+        throw new IOException("The provided alias cannot be resolved");
+      }
+      String pass = new String(cred);
+      return pass;
+    } catch (InvocationTargetException ite) {
+      throw new RuntimeException("Error resolving password "
+        + " from the credential providers ", ite.getTargetException());
+    } catch (IllegalAccessException iae) {
+      throw new RuntimeException("Error invoking the credential provider method",
+        iae);
+    }
+  }
+  /**
+   * Test utility to create an entry
+   */
+  public static void createCredentialEntry(Configuration conf,
+
+    String alias, String credential) throws IOException {
+
+    if (!isProviderAvailable()) {
+      throw new RuntimeException("CredentialProvider facility not available "
+        + "in the hadoop environment");
+    }
+
+
+    try {
+      List<?> result = (List<?>)
+        methGetProviders.invoke(null, new Object[] { conf });
+      Object provider = result.get(0);
+      LOG.debug("Using credential provider " + provider);
+
+      methCreateCredEntry.invoke(provider, new Object[] {
+        alias, credential.toCharArray() });
+      methFlush.invoke(provider, new Object[] {});
+    } catch (InvocationTargetException ite) {
+      throw new RuntimeException("Error creating credential entry "
+        + " using the credentail provider", ite.getTargetException());
+    } catch (IllegalAccessException iae) {
+      throw new RuntimeException("Error accessing the credential create method",
+        iae);
+    }
+  }
+}

src/java/org/apache/sqoop/util/password/CredentialProviderPasswordLoader.java

@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.util.password;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * A password loader that loads an credential provider alias.
+ * The alias is  resolved using the Hadoop credential provider facilitity
+ * if available.
+ */
+public class CredentialProviderPasswordLoader extends FilePasswordLoader {
+  public static final Log LOG =
+    LogFactory.getLog(CredentialProviderPasswordLoader.class.getName());
+
+  /**
+   * If credential provider is available (made available as part of 2.6.0 and
+   * 3.0, then use the credential provider to get the password. Else throw an
+   * exception saying this provider is not available.
+   */
+  @Override
+  public String loadPassword(String p, Configuration configuration)
+    throws IOException {
+    if (!CredentialProviderHelper.isProviderAvailable()) {
+      throw new IOException("CredentialProvider facility not available "
+        + "in the hadoop environment used");
+    }
+    LOG.debug("Fetching alias from the specified path: " + p);
+    Path path = new Path(p);
+    FileSystem fs = path.getFileSystem(configuration);
+
+    // Not closing FileSystem object because of SQOOP-1226
+    verifyPath(fs, path);
+    String alias = new String(readBytes(fs, path));
+    String pass = CredentialProviderHelper.resolveAlias(configuration, alias);
+    return pass;
+  }
+}

src/java/org/apache/sqoop/util/password/FilePasswordLoader.java

@@ -48,11 +48,13 @@ public class FilePasswordLoader extends PasswordLoader {
    */
   protected void verifyPath(FileSystem fs, Path path) throws IOException {
     if (!fs.exists(path)) {
-      throw new IOException("The password file does not exist! " + path);
+      throw new IOException("The provided password file " + path
+        + " does not exist!");
     }
 
     if (!fs.isFile(path)) {
-      throw new IOException("The password file cannot be a directory! " + path);
+      throw new IOException("The provided password file " + path
+        + " is a directory!");
     }
   }
 

src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java

@@ -21,6 +21,7 @@
 import com.cloudera.sqoop.SqoopOptions;
 import com.cloudera.sqoop.testutil.BaseSqoopTestCase;
 import com.cloudera.sqoop.testutil.CommonArgs;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -29,13 +30,17 @@
 import org.apache.sqoop.mapreduce.db.DBConfiguration;
 import org.apache.sqoop.tool.BaseSqoopTool;
 import org.apache.sqoop.tool.ImportTool;
+import org.apache.sqoop.util.password.CredentialProviderHelper;
+import org.apache.sqoop.util.password.CredentialProviderPasswordLoader;
 import org.apache.sqoop.util.password.CryptoFileLoader;
+import org.apache.sqoop.util.password.PasswordLoader;
 
 import javax.crypto.Cipher;
 import javax.crypto.SecretKey;
 import javax.crypto.SecretKeyFactory;
 import javax.crypto.spec.PBEKeySpec;
 import javax.crypto.spec.SecretKeySpec;
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -93,9 +98,10 @@ public void testPasswordFileDoesNotExist() throws Exception {
       SqoopOptions opts = getSqoopOptions(conf);
       ImportTool importTool = new ImportTool();
       importTool.parseArguments(argv, conf, opts, true);
-      fail("The password file does not exist! ");
+      fail("The password file does not exist!");
     } catch (Exception e) {
-      assertTrue(e.getMessage().contains("The password file does not exist!"));
+      assertTrue(e.getMessage().matches(".*The provided password file "
+        + ".* does not exist!"));
     }
   }
 
@@ -110,10 +116,10 @@ public void testPasswordFileIsADirectory() throws Exception {
       SqoopOptions opts = getSqoopOptions(conf);
       ImportTool importTool = new ImportTool();
       importTool.parseArguments(argv, conf, opts, true);
-      fail("The password file cannot be a directory! ");
+      fail("The password file cannot be a directory!");
     } catch (Exception e) {
-      assertTrue(e.getMessage().contains("The password file cannot "
+      assertTrue(e.getMessage().matches(".*The provided password file .*"
-        + "be a directory!"));
+        + " is a directory!"));
     }
   }
 
@@ -137,10 +143,11 @@ public void testBothPasswordOptions() throws Exception {
       SqoopOptions out = importTool.parseArguments(argv, conf, in, true);
       assertNotNull(out.getPassword());
       importTool.validateOptions(out);
-      fail("Either password or passwordPath must be specified but not both.");
+      fail("Only one of password, password "
+          + "alias or path to a password file must be specified.");
     } catch (Exception e) {
-      assertTrue(e.getMessage().contains("Either password or path to a "
+      assertTrue(e.getMessage().contains("Only one of password, password "
-        + "password file must be specified but not both"));
+          + "alias or path to a password file must be specified."));
     }
   }
 
@@ -351,6 +358,88 @@ public void testCryptoFileLoader() throws Exception {
     }
   }
 
+  public void testCredentialProviderLoader() throws Exception {
+    CredentialProviderPasswordLoader pl =
+        new CredentialProviderPasswordLoader();
+
+    if (!CredentialProviderHelper.isProviderAvailable()) {
+      LOG.info("CredentialProvider facility not available "
+        + "in the hadoop environment used");
+    } else {
+      String alias = "super.secret.alias";
+      String pw = "super.secret.password";
+
+      String passwordFilePath = TEMP_BASE_DIR + ".pwd";
+      String jksFile = "creds.jks";
+      createTempFile(passwordFilePath);
+      writeToFile(passwordFilePath, alias.getBytes());
+      File credDir = new File(".");
+
+      Configuration conf = getConf();
+      String ourUrl =  CredentialProviderHelper.SCHEME_NAME +
+        "://file/" + credDir.getAbsolutePath() + "/" + jksFile;
+      File file = new File(credDir, jksFile);
+      file.delete();
+      conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH,
+        ourUrl);
+      CredentialProviderHelper.createCredentialEntry(conf, alias, pw);
+
+      conf.set("org.apache.sqoop.credentials.loader.class",
+        CredentialProviderPasswordLoader.class.getCanonicalName());
+
+      ArrayList<String> extraArgs = new ArrayList<String>();
+      extraArgs.add("--username");
+      extraArgs.add("username");
+      extraArgs.add("--password-file");
+      extraArgs.add(passwordFilePath);
+      String[] commonArgs = getCommonArgs(false, extraArgs);
+
+      SqoopOptions in = getSqoopOptions(conf);
+      ImportTool importTool = new ImportTool();
+
+      SqoopOptions out = importTool.parseArguments(commonArgs, conf, in, true);
+      assertEquals(pw, pl.loadPassword(passwordFilePath, conf));
+      assertEquals(pw, out.getPassword());
+    }
+  }
+
+  public void testPasswordAliasOption() throws Exception {
+    CredentialProviderPasswordLoader pl =
+        new CredentialProviderPasswordLoader();
+
+    if (!CredentialProviderHelper.isProviderAvailable()) {
+      LOG.info("CredentialProvider facility not available "
+        + "in the hadoop environment used");
+    } else {
+      String alias = "super.secret.alias";
+      String pw = "super.secret.password";
+      String jksFile = "creds.jks";
+      File credDir = new File(".");
+
+      Configuration conf = getConf();
+      String ourUrl =  CredentialProviderHelper.SCHEME_NAME +
+        "://file/" + credDir.getAbsolutePath() + "/" + jksFile;
+      File file = new File(credDir, jksFile);
+      file.delete();
+      conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH,
+        ourUrl);
+      CredentialProviderHelper.createCredentialEntry(conf, alias, pw);
+
+      ArrayList<String> extraArgs = new ArrayList<String>();
+      extraArgs.add("--username");
+      extraArgs.add("username");
+      extraArgs.add("--password-alias");
+      extraArgs.add(alias);
+      String[] commonArgs = getCommonArgs(false, extraArgs);
+
+      SqoopOptions in = getSqoopOptions(conf);
+      ImportTool importTool = new ImportTool();
+
+      SqoopOptions out = importTool.parseArguments(commonArgs, conf, in, true);
+      assertEquals(pw, out.getPassword());
+    }
+  }
+
   public void executeCipherTest(String password, String passphrase, String cipher, int keySize) throws Exception {
     LOG.info("Using cipher: " + cipher + " with keySize " + keySize + " and passphrase " + passphrase );
     String passwordFilePath = TEMP_BASE_DIR + ".pwd";