From 08eb5bdc405c22a25cf680390bbd438f6513199c Mon Sep 17 00:00:00 2001 From: Szabolcs Vasas Date: Fri, 23 Nov 2018 15:15:02 +0100 Subject: [PATCH] SQOOP-3410: Test S3 import with fs.s3a.security.credential.provider.path (Boglarka Egyed via Szabolcs Vasas) --- src/docs/user/s3.txt | 6 ++++-- .../password/CredentialProviderHelper.java | 6 ++++-- .../TestPassingSecurePassword.java | 4 ++-- .../TestS3ImportWithHadoopCredProvider.java | 21 ++++++++++++++++++- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/docs/user/s3.txt b/src/docs/user/s3.txt index 6ff828c4..e3d97717 100644 --- a/src/docs/user/s3.txt +++ b/src/docs/user/s3.txt @@ -173,8 +173,10 @@ https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html#Pr For a guide to the Hadoop Credential Provider API please see the Hadoop documentation at https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html. -After creating a credential file with the credential entries the URL to the provider can be set via the -+hadoop.security.credential.provider.path+ property. +After creating a credential file with the credential entries the URL to the provider can be set via either the ++hadoop.security.credential.provider.path+ or the +fs.s3a.security.credential.provider.path+ property. For learning +more about the precedence of these please see the Hadoop AWS documentation at +https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html#Configure_the_hadoop.security.credential.provider.path_property. Hadoop Credential Provider is often protected by password supporting three options: diff --git a/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java b/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java index 4e79f0ae..e659def1 100644 --- a/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java +++ b/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java @@ -83,8 +83,10 @@ public class CredentialProviderHelper { // Should track what is specified in JavaKeyStoreProvider class. public static final String SCHEME_NAME = "jceks"; // Should track what is in CredentialProvider class. - public static final String CREDENTIAL_PROVIDER_PATH = + public static final String HADOOP_CREDENTIAL_PROVIDER_PATH = "hadoop.security.credential.provider.path"; + public static final String S3A_CREDENTIAL_PROVIDER_PATH = + "fs.s3a.security.credential.provider.path"; public static final String CREDENTIAL_PROVIDER_PASSWORD_FILE = "hadoop.security.credstore.java-keystore-provider.password-file"; @@ -103,7 +105,7 @@ public static boolean isProviderAvailable() { public static String resolveAlias(Configuration conf, String alias) throws IOException { LOG.debug("Resolving alias with credential provider path set to " - + conf.get(CREDENTIAL_PROVIDER_PATH)); + + conf.get(HADOOP_CREDENTIAL_PROVIDER_PATH)); try { char[] cred = (char[]) methGetPassword.invoke(conf, new Object[] { alias }); diff --git a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java index dca3195b..4a8e7dc4 100644 --- a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java +++ b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java @@ -397,7 +397,7 @@ public void testCredentialProviderLoader() throws Exception { "://file/" + credDir.getAbsolutePath() + "/" + jksFile; File file = new File(credDir, jksFile); file.delete(); - conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, + conf.set(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH, ourUrl); CredentialProviderHelper.createCredentialEntry(conf, alias, pw); @@ -439,7 +439,7 @@ public void testPasswordAliasOption() throws Exception { "://file/" + credDir.getAbsolutePath() + "/" + jksFile; File file = new File(credDir, jksFile); file.delete(); - conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, + conf.set(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH, ourUrl); CredentialProviderHelper.createCredentialEntry(conf, alias, pw); diff --git a/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java b/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java index e1d7cbda..534d5a96 100644 --- a/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java +++ b/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java @@ -32,6 +32,7 @@ import org.apache.sqoop.testutil.S3CredentialGenerator; import org.apache.sqoop.testutil.S3TestUtils; import org.apache.sqoop.testutil.TextFileTestUtils; +import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory; import org.apache.sqoop.util.password.CredentialProviderHelper; import org.junit.After; import org.junit.AfterClass; @@ -43,18 +44,36 @@ import org.junit.contrib.java.lang.system.EnvironmentVariables; import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.util.Arrays; import static junit.framework.TestCase.fail; @Category(S3Test.class) +@RunWith(Parameterized.class) +@Parameterized.UseParametersRunnerFactory(BlockJUnit4ClassRunnerWithParametersFactory.class) public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase { + + @Parameterized.Parameters(name = "credentialProviderPathProperty = {0}") + public static Iterable parameters() { + return Arrays.asList(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH, + CredentialProviderHelper.S3A_CREDENTIAL_PROVIDER_PATH); + } + public static final Log LOG = LogFactory.getLog( TestS3ImportWithHadoopCredProvider.class.getName()); + private String credentialProviderPathProperty; + + public TestS3ImportWithHadoopCredProvider(String credentialProviderPathProperty) { + this.credentialProviderPathProperty = credentialProviderPathProperty; + } + private static S3CredentialGenerator s3CredentialGenerator; private static String providerPathDefault; @@ -156,7 +175,7 @@ public void testCredentialProviderWithNoPwdFileFails() throws Exception { private String[] getArgs(String providerPath, boolean withPwdFile, String pwdFile) { ArgumentArrayBuilder builder = S3TestUtils.getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(this); - builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, providerPath); + builder.withProperty(credentialProviderPathProperty, providerPath); if (withPwdFile) { builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, pwdFile); }