SQOOP-3410: Test S3 import with fs.s3a.security.credential.provider.path

(Boglarka Egyed via Szabolcs Vasas)

src/docs/user/s3.txt

@@ -173,8 +173,10 @@ https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html#Pr
 For a guide to the Hadoop Credential Provider API please see the Hadoop documentation at
 https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html.
 
-After creating a credential file with the credential entries the URL to the provider can be set via the
+After creating a credential file with the credential entries the URL to the provider can be set via either the
-+hadoop.security.credential.provider.path+ property.
++hadoop.security.credential.provider.path+ or the +fs.s3a.security.credential.provider.path+ property. For learning
+more about the precedence of these please see the Hadoop AWS documentation at
+https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html#Configure_the_hadoop.security.credential.provider.path_property.
 
 Hadoop Credential Provider is often protected by password supporting three options:
 

src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java

@@ -83,8 +83,10 @@ public class CredentialProviderHelper {
   // Should track what is specified in JavaKeyStoreProvider class.
   public static final String SCHEME_NAME = "jceks";
   // Should track what is in CredentialProvider class.
-  public static final String CREDENTIAL_PROVIDER_PATH =
+  public static final String HADOOP_CREDENTIAL_PROVIDER_PATH =
     "hadoop.security.credential.provider.path";
+  public static final String S3A_CREDENTIAL_PROVIDER_PATH =
+          "fs.s3a.security.credential.provider.path";
   public static final String CREDENTIAL_PROVIDER_PASSWORD_FILE =
           "hadoop.security.credstore.java-keystore-provider.password-file";
 
@@ -103,7 +105,7 @@ public static boolean isProviderAvailable() {
   public static String resolveAlias(Configuration conf, String alias)
     throws IOException {
     LOG.debug("Resolving alias with credential provider path set to "
-      + conf.get(CREDENTIAL_PROVIDER_PATH));
+      + conf.get(HADOOP_CREDENTIAL_PROVIDER_PATH));
     try {
       char[] cred = (char[])
         methGetPassword.invoke(conf, new Object[] { alias });

src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java

@@ -397,7 +397,7 @@ public void testCredentialProviderLoader() throws Exception {
         "://file/" + credDir.getAbsolutePath() + "/" + jksFile;
       File file = new File(credDir, jksFile);
       file.delete();
-      conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH,
+      conf.set(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
         ourUrl);
       CredentialProviderHelper.createCredentialEntry(conf, alias, pw);
 
@@ -439,7 +439,7 @@ public void testPasswordAliasOption() throws Exception {
         "://file/" + credDir.getAbsolutePath() + "/" + jksFile;
       File file = new File(credDir, jksFile);
       file.delete();
-      conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH,
+      conf.set(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
         ourUrl);
       CredentialProviderHelper.createCredentialEntry(conf, alias, pw);
 

src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java

@@ -32,6 +32,7 @@
 import org.apache.sqoop.testutil.S3CredentialGenerator;
 import org.apache.sqoop.testutil.S3TestUtils;
 import org.apache.sqoop.testutil.TextFileTestUtils;
+import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory;
 import org.apache.sqoop.util.password.CredentialProviderHelper;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -43,18 +44,36 @@
 import org.junit.contrib.java.lang.system.EnvironmentVariables;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.util.Arrays;
 
 import static junit.framework.TestCase.fail;
 
 @Category(S3Test.class)
+@RunWith(Parameterized.class)
+@Parameterized.UseParametersRunnerFactory(BlockJUnit4ClassRunnerWithParametersFactory.class)
 public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase {
+
+    @Parameterized.Parameters(name = "credentialProviderPathProperty = {0}")
+    public static Iterable<? extends Object> parameters() {
+        return Arrays.asList(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
+            CredentialProviderHelper.S3A_CREDENTIAL_PROVIDER_PATH);
+    }
+
     public static final Log LOG = LogFactory.getLog(
             TestS3ImportWithHadoopCredProvider.class.getName());
 
+    private String credentialProviderPathProperty;
+
+    public TestS3ImportWithHadoopCredProvider(String credentialProviderPathProperty) {
+        this.credentialProviderPathProperty = credentialProviderPathProperty;
+    }
+
     private static S3CredentialGenerator s3CredentialGenerator;
 
     private static String providerPathDefault;
@@ -156,7 +175,7 @@ public void testCredentialProviderWithNoPwdFileFails() throws Exception {
     private String[] getArgs(String providerPath, boolean withPwdFile, String pwdFile) {
         ArgumentArrayBuilder builder = S3TestUtils.getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(this);
 
-        builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, providerPath);
+        builder.withProperty(credentialProviderPathProperty, providerPath);
         if (withPwdFile) {
             builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, pwdFile);
         }