SQOOP-1223. Enhance the password file capability to enable plugging-in custom loaders.

(Jarek Jarcec Cecho via Hari Shreedharan)

src/java/org/apache/sqoop/SqoopOptions.java

@@ -613,8 +613,7 @@ private void loadPasswordProperty(Properties props) {
     passwordFilePath = props.getProperty("db.password.file");
     if (passwordFilePath != null) {
       try {
-        password = CredentialsUtil.fetchPasswordFromFile(
+        password = CredentialsUtil.fetchPasswordFromLoader(passwordFilePath, getConf());
-                getConf(), passwordFilePath);
         return; // short-circuit
       } catch (IOException e) {
         throw new RuntimeException("Unable to fetch password from file.", e);

src/java/org/apache/sqoop/tool/BaseSqoopTool.java

@@ -26,7 +26,6 @@
 import java.util.Arrays;
 import java.util.Properties;
 
-import com.cloudera.sqoop.util.ImportException;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
@@ -878,7 +877,9 @@ private void applyCredentialsOptions(CommandLine in, SqoopOptions out)
       try {
         out.setPasswordFilePath(in.getOptionValue(PASSWORD_PATH_ARG));
         // apply password from file into password in options
-        out.setPassword(CredentialsUtil.fetchPasswordFromFile(out));
+        out.setPassword(CredentialsUtil.fetchPassword(out));
+        // And allow the PasswordLoader to clean up any sensitive properties
+        CredentialsUtil.cleanUpSensitiveProperties(out.getConf());
       } catch (IOException ex) {
         LOG.warn("Failed to load connection parameter file", ex);
         throw new InvalidOptionsException(

src/java/org/apache/sqoop/util/CredentialsUtil.java

@@ -21,64 +21,91 @@
 package org.apache.sqoop.util;
 
 import com.cloudera.sqoop.SqoopOptions;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.sqoop.util.password.FilePasswordLoader;
-import org.apache.hadoop.fs.Path;
+import org.apache.sqoop.util.password.PasswordLoader;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
 
 /**
  * A utility class for fetching passwords from a file.
  */
 public final class CredentialsUtil {
 
-  public static final Log LOG = LogFactory.getLog(
+  /**
-    CredentialsUtil.class.getName());
+   * Property for specifying which loader should be used to fetch the password.
+   */
+  private static String PROPERTY_LOADER_CLASSS = "org.apache.sqoop.credentials.loader.class";
+
+  /**
+   * The default loader is a FilePasswordLoader that will fetch the password from a file.
+   */
+  private static String DEFAULT_PASSWORD_LOADER = FilePasswordLoader.class.getCanonicalName();
+
+  public static final Log LOG = LogFactory.getLog(CredentialsUtil.class.getName());
 
   private CredentialsUtil() {
   }
 
-  public static String fetchPasswordFromFile(SqoopOptions options)
+  /**
+   * Return password that was specified (either on command line or via other facilities).
+   *
+   * @param options Sqoop Options
+   * @return Password
+   * @throws IOException
+   */
+  public static String fetchPassword(SqoopOptions options)
     throws IOException {
     String passwordFilePath = options.getPasswordFilePath();
     if (passwordFilePath == null) {
       return options.getPassword();
     }
 
-    return fetchPasswordFromFile(options.getConf(), passwordFilePath);
+    return fetchPasswordFromLoader(options.getPasswordFilePath(), options.getConf());
   }
 
-  public static String fetchPasswordFromFile(Configuration conf,
+  /**
-                                             String passwordFilePath)
+   * Return password via --password-file argument.
-    throws IOException {
+   *
-    LOG.debug("Fetching password from specified path: " + passwordFilePath);
+   * Given loader can be overridden using PROPERTY_LOADER_CLASSS.
-    Path path = new Path(passwordFilePath);
+   *
-    FileSystem fs = path.getFileSystem(conf);
+   * @param path Path with the password file.
+   * @param conf Configuration
+   * @return Password
+   * @throws IOException
+   */
+  public static String fetchPasswordFromLoader(String path, Configuration conf) throws IOException {
+    PasswordLoader loader = getLoader(conf);
+    return loader.loadPassword(path, conf);
+  }
 
-    if (!fs.exists(path)) {
+  /**
-      throw new IOException("The password file does not exist! "
+   * Remove any potentially sensitive information from the configuration object.
-        + passwordFilePath);
+   *
-    }
+   * @param configuration Associated configuration object.
+   * @throws IOException
+   */
+  public static void cleanUpSensitiveProperties(Configuration configuration) throws IOException {
+    PasswordLoader loader = getLoader(configuration);
+    loader.cleanUpConfiguration(configuration);
+  }
 
-    if (!fs.isFile(path)) {
+  /**
-      throw new IOException("The password file cannot be a directory! "
+   * Instantiate configured PasswordLoader class.
-        + passwordFilePath);
+   *
-    }
+   * @param configuration Associated configuration object.
+   * @return
+   * @throws IOException
+   */
+  public static PasswordLoader getLoader(Configuration configuration) throws IOException {
+    String loaderClass = configuration.get(PROPERTY_LOADER_CLASSS, DEFAULT_PASSWORD_LOADER);
 
-    InputStream is = fs.open(path);
-    StringWriter writer = new StringWriter();
     try {
-      IOUtils.copy(is, writer);
+      return (PasswordLoader) Class.forName(loaderClass).newInstance();
-      return writer.toString();
+    } catch (Exception e) {
-    } finally {
+      throw new IOException(e);
-      IOUtils.closeQuietly(is);
-      IOUtils.closeQuietly(writer);
-      fs.close();
     }
   }
 }

src/java/org/apache/sqoop/util/password/CryptoFileLoader.java

@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.util.password;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import javax.crypto.Cipher;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+
+
+/**
+ * Example implementation of "advanced" file loader that will read password from
+ * encrypted file. Please note that this method is merely obfuscating the password,
+ * as malicious user will be able to retrieve the password if he intercepts the
+ * Sqoop commands. He won't be able to get it if he will just get the password
+ * file though. Current implementation is limited to ECB and is not supporting other
+ * methods.
+ *
+ * Example usage:
+ *   sqoop import  \
+ *     -Dorg.apache.sqoop.credentials.loader.class=org.apache.sqoop.util.password.CryptoFileLoader \
+ *     -Dorg.apache.sqoop.credentials.loader.crypto.passphrase=sqooppass \
+ *     --connect ...
+ */
+public class CryptoFileLoader extends FilePasswordLoader {
+
+  /**
+   * Crypto algorithm that should be used.
+   *
+   * List of available ciphers is for example available here:
+   * http://docs.oracle.com/javase/7/docs/api/javax/crypto/Cipher.html
+   */
+  private static String PROPERTY_CRYPTO_ALG = "org.apache.sqoop.credentials.loader.crypto.alg";
+
+  /**
+   * Salt that should be used.
+   *
+   * Some algorithms are requiring salt to be present.
+   */
+  private static String PROPERTY_CRYPTO_SALT = "org.apache.sqoop.credentials.loader.crypto.salt";
+
+  /**
+   * Iterations argument for creating key.
+   */
+  private static String PROPERTY_CRYPTO_ITERATIONS = "org.apache.sqoop.credentials.loader.crypto.iterations";
+
+  /**
+   * Length of the key (driven by the used algorithm).
+   */
+  private static String PROPERTY_CRYPTO_KEY_LEN = "org.apache.sqoop.credentials.loader.crypto.salt.key.len";
+
+  /**
+   * Passphrase (encryption password).
+   */
+  private static String PROPERTY_CRYPTO_PASSPHRASE = "org.apache.sqoop.credentials.loader.crypto.passphrase";
+
+  /**
+   * Default is AES in electronic code book with padding.
+   */
+  private static String DEFAULT_ALG = "AES/ECB/PKCS5Padding";
+
+  /**
+   * Default salt is not much secure, use your own!
+   */
+  private static String DEFAULT_SALT = "SALT";
+
+  /**
+   * Iterate 10000 times by default.
+   */
+  private static int DEFAULT_ITERATIONS = 10000;
+
+  /**
+   * One of valid key sizes for default algorithm (AES).
+   */
+  private static int DEFAULT_KEY_LEN = 128;
+
+  @Override
+  public String loadPassword(String p, Configuration configuration) throws IOException {
+    LOG.debug("Fetching password from specified path: " + p);
+    Path path = new Path(p);
+    FileSystem fs = path.getFileSystem(configuration);
+
+    byte [] encrypted;
+    try {
+      verifyPath(fs, path);
+      encrypted = readBytes(fs, path);
+    } finally {
+      fs.close();
+    }
+
+    String passPhrase = configuration.get(PROPERTY_CRYPTO_PASSPHRASE);
+    if(passPhrase == null) {
+      throw new IOException("Passphrase is missing in property " + PROPERTY_CRYPTO_PASSPHRASE);
+    }
+
+    String alg = configuration.get(PROPERTY_CRYPTO_ALG, DEFAULT_ALG);
+    String algOnly = alg.split("/")[0];
+    String salt = configuration.get(PROPERTY_CRYPTO_SALT, DEFAULT_SALT);
+    int iterations = configuration.getInt(PROPERTY_CRYPTO_ITERATIONS, DEFAULT_ITERATIONS);
+    int keyLen = configuration.getInt(PROPERTY_CRYPTO_KEY_LEN, DEFAULT_KEY_LEN);
+
+    SecretKeyFactory factory = null;
+    try {
+      factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
+    } catch (NoSuchAlgorithmException e) {
+      throw new IOException("Can't load SecretKeyFactory", e);
+    }
+
+    SecretKeySpec key = null;
+    try {
+      key = new SecretKeySpec(factory.generateSecret(new PBEKeySpec(passPhrase.toCharArray(), salt.getBytes(), iterations, keyLen)).getEncoded(), algOnly);
+    } catch (Exception e) {
+      throw new IOException("Can't generate secret key", e);
+    }
+
+    Cipher crypto = null;
+
+    try {
+      crypto = Cipher.getInstance(alg);
+    } catch (Exception e) {
+      throw new IOException("Can't initialize the decryptor", e);
+    }
+
+    byte[] decryptedBytes;
+
+    try {
+      crypto.init(Cipher.DECRYPT_MODE, key);
+      decryptedBytes = crypto.doFinal(encrypted);
+    } catch (Exception e) {
+      throw new IOException("Can't decrypt the password", e);
+    }
+
+    return new String(decryptedBytes);
+  }
+
+  @Override
+  public void cleanUpConfiguration(Configuration configuration) {
+    configuration.unset(PROPERTY_CRYPTO_PASSPHRASE);
+    configuration.unset(PROPERTY_CRYPTO_SALT);
+    configuration.unset(PROPERTY_CRYPTO_KEY_LEN);
+    configuration.unset(PROPERTY_CRYPTO_ITERATIONS);
+  }
+}

src/java/org/apache/sqoop/util/password/FilePasswordLoader.java

@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.util.password;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import java.io.IOException;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import java.io.InputStream;
+
+/**
+ * Simple password loader that will load file from file system.
+ *
+ * The file system can be either local or any remote implementation supported
+ * by your Hadoop installation. As the password needs to be stored in the file
+ * in a clear text, the password file should have very restrictive permissions
+ * (400).
+ */
+public class FilePasswordLoader extends PasswordLoader {
+
+  public static final Log LOG = LogFactory.getLog(FilePasswordLoader.class.getName());
+
+  /**
+   * Verify that given path leads to a file that we can read.
+   *
+   * @param fs Associated FileSystem
+   * @param path Path
+   * @throws IOException
+   */
+  protected void verifyPath(FileSystem fs, Path path) throws IOException {
+    if (!fs.exists(path)) {
+      throw new IOException("The password file does not exist! " + path);
+    }
+
+    if (!fs.isFile(path)) {
+      throw new IOException("The password file cannot be a directory! " + path);
+    }
+  }
+
+  /**
+   * Read bytes from given file.
+   *
+   * @param fs Associated FileSystem
+   * @param path Path
+   * @return
+   * @throws IOException
+   */
+  protected byte [] readBytes(FileSystem fs, Path path) throws IOException {
+    InputStream is = fs.open(path);
+    try {
+      return IOUtils.toByteArray(is);
+    } finally {
+      IOUtils.closeQuietly(is);
+    }
+  }
+
+  @Override
+  public String loadPassword(String p, Configuration configuration) throws IOException {
+    LOG.debug("Fetching password from specified path: " + p);
+    Path path = new Path(p);
+    FileSystem fs = path.getFileSystem(configuration);
+
+    try {
+      verifyPath(fs, path);
+      return new String(readBytes(fs, path));
+    } finally {
+      fs.close();
+    }
+  }
+}

src/java/org/apache/sqoop/util/password/PasswordLoader.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.util.password;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.IOException;
+
+/**
+ * Abstract class for describing password loader.
+ */
+abstract public class PasswordLoader {
+
+  /**
+   * Load password from given path.
+   *
+   * @param path Path to load password from.
+   * @param configuration Configuration object.
+   * @return Password
+   * @throws IOException
+   */
+  public abstract String loadPassword(String path, Configuration configuration) throws IOException;
+
+  /**
+   * Callback that allows to clean up configuration properties out of the
+   * configuration object that will be passed down to the mapreduce framework.
+   *
+   * It's defined by the Sqoop framework that after calling cleanUpConfiguration(),
+   * method loadPassword() will never be called again.
+   */
+  public void cleanUpConfiguration(Configuration configuration) {
+    // Default implementation is empty
+  }
+}

src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java

@@ -29,10 +29,17 @@
 import org.apache.sqoop.mapreduce.db.DBConfiguration;
 import org.apache.sqoop.tool.BaseSqoopTool;
 import org.apache.sqoop.tool.ImportTool;
+import org.apache.sqoop.util.password.CryptoFileLoader;
 
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.SecretKeySpec;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
 import java.sql.Connection;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -314,18 +321,93 @@ private String[] getCommonArgs(boolean includeHadoopFlags,
     return args.toArray(new String[0]);
   }
 
+  public void testCryptoFileLoader() throws Exception {
+    // Current implementation is limited to ECB mode
+    Object[][] ciphers = {
+//      {"AES/CBC/NoPadding", 128},
+//      {"AES/CBC/PKCS5Padding", 128},
+      {"AES/ECB/NoPadding", 128},
+      {"AES/ECB/PKCS5Padding", 128},
+//      {"DES/CBC/NoPadding", 56},
+//      {"DES/CBC/PKCS5Padding", 56},
+      {"DES/ECB/NoPadding", 64},
+      {"DES/ECB/PKCS5Padding", 64},
+//      {"DESede/CBC/NoPadding", 168},
+//      {"DESede/CBC/PKCS5Padding", 168},
+      {"DESede/ECB/NoPadding", 192},
+      {"DESede/ECB/PKCS5Padding", 192}
+    };
+
+    String [] passphrases = {
+      "Simple  password",
+      "!@#$%^&*()_+<>?:"
+    };
+
+    // Execute all ciphers with all pass phrases
+    for(Object [] cipher : ciphers) {
+      for(String pass : passphrases) {
+        executeCipherTest(pass, pass, (String)cipher[0], (Integer)cipher[1]);
+      }
+    }
+  }
+
+  public void executeCipherTest(String password, String passphrase, String cipher, int keySize) throws Exception {
+    LOG.info("Using cipher: " + cipher + " with keySize " + keySize + " and passphrase " + passphrase );
+    String passwordFilePath = TEMP_BASE_DIR + ".pwd";
+    createTempFile(passwordFilePath);
+    writeToFile(passwordFilePath, encryptPassword(password, passphrase, cipher, 10000, keySize));
+    LOG.info("Generated encrypted password file in: " + passwordFilePath);
+
+    ArrayList<String> extraArgs = new ArrayList<String>();
+    extraArgs.add("--username");
+    extraArgs.add("username");
+    extraArgs.add("--password-file");
+    extraArgs.add(passwordFilePath);
+    String[] commonArgs = getCommonArgs(false, extraArgs);
+
+    Configuration conf = getConf();
+    conf.set("org.apache.sqoop.credentials.loader.class", CryptoFileLoader.class.getCanonicalName());
+    conf.set("org.apache.sqoop.credentials.loader.crypto.alg", cipher);
+    conf.set("org.apache.sqoop.credentials.loader.crypto.passphrase", passphrase);
+    conf.setInt("org.apache.sqoop.credentials.loader.crypto.salt.key.len", keySize);
+
+    SqoopOptions in = getSqoopOptions(conf);
+    ImportTool importTool = new ImportTool();
+
+    SqoopOptions out = importTool.parseArguments(commonArgs, conf, in, true);
+    assertNotNull(out.getPasswordFilePath());
+    assertNotNull(out.getPassword());
+    assertEquals(passphrase, out.getPassword());
+  }
+
+  private byte[] encryptPassword(String password, String passPhrase, String alg, int iterations, int keySize) throws Exception {
+    String algOnly = alg.split("/")[0];
+
+    SecretKeyFactory factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
+    SecretKey secKey =  factory.generateSecret(new PBEKeySpec(passPhrase.toCharArray(), "SALT".getBytes(), iterations, keySize));
+    SecretKeySpec key = new SecretKeySpec(secKey.getEncoded(), algOnly);
+
+    Cipher crypto = Cipher.getInstance(alg);
+    crypto.init(Cipher.ENCRYPT_MODE, key);
+
+    return crypto.doFinal(password.getBytes());
+  }
+
   private void createTempFile(String filePath) throws IOException {
     File pwdFile = new File(filePath);
     pwdFile.createNewFile();
   }
 
-  private void writeToFile(String filePath, String contents)
+  private void writeToFile(String filePath, String contents) throws IOException {
-    throws IOException {
+    writeToFile(filePath, contents.getBytes());
+  }
+
+  private void writeToFile(String filePath, byte [] contents) throws IOException {
     File pwdFile = new File(filePath);
     FileOutputStream fos = null;
     try {
       fos = new FileOutputStream(pwdFile);
-      fos.write(contents.getBytes());
+      fos.write(contents);
     } finally {
       if (fos != null) {
         fos.close();