github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-02 17:22:25 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3410: Test S3 import with fs.s3a.security.credential.provider.path

Browse Source 
(Boglarka Egyed via Szabolcs Vasas)
This commit is contained in:
Szabolcs Vasas 2018-11-23 15:15:02 +01:00
parent 2bf6f3ccd0
commit 08eb5bdc40
4 changed files with 30 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/docs/user/s3.txt
View File

@ -173,8 +173,10 @@ https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html#Pr
For a guide to the Hadoop Credential Provider API please see the Hadoop documentation at
https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html.
After creating a credential file with the credential entries the URL to the provider can be set via the
+hadoop.security.credential.provider.path+ property.
After creating a credential file with the credential entries the URL to the provider can be set via either the
+hadoop.security.credential.provider.path+ or the +fs.s3a.security.credential.provider.path+ property. For learning
more about the precedence of these please see the Hadoop AWS documentation at
https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html#Configure_the_hadoop.security.credential.provider.path_property.
Hadoop Credential Provider is often protected by password supporting three options:

6
src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
View File

@ -83,8 +83,10 @@ public class CredentialProviderHelper {
// Should track what is specified in JavaKeyStoreProvider class.
public static final String SCHEME_NAME = "jceks";
// Should track what is in CredentialProvider class.
public static final String CREDENTIAL_PROVIDER_PATH =
public static final String HADOOP_CREDENTIAL_PROVIDER_PATH =
"hadoop.security.credential.provider.path";
public static final String S3A_CREDENTIAL_PROVIDER_PATH =
"fs.s3a.security.credential.provider.path";
public static final String CREDENTIAL_PROVIDER_PASSWORD_FILE =
"hadoop.security.credstore.java-keystore-provider.password-file";
@ -103,7 +105,7 @@ public static boolean isProviderAvailable() {
public static String resolveAlias(Configuration conf, String alias)
throws IOException {
LOG.debug("Resolving alias with credential provider path set to "
+ conf.get(CREDENTIAL_PROVIDER_PATH));
+ conf.get(HADOOP_CREDENTIAL_PROVIDER_PATH));
try {
char[] cred = (char[])
methGetPassword.invoke(conf, new Object[] { alias });

4
src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java
View File

@ -397,7 +397,7 @@ public void testCredentialProviderLoader() throws Exception {
"://file/" + credDir.getAbsolutePath() + "/" + jksFile;
File file = new File(credDir, jksFile);
file.delete();
conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH,
conf.set(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
ourUrl);
CredentialProviderHelper.createCredentialEntry(conf, alias, pw);
@ -439,7 +439,7 @@ public void testPasswordAliasOption() throws Exception {
"://file/" + credDir.getAbsolutePath() + "/" + jksFile;
File file = new File(credDir, jksFile);
file.delete();
conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH,
conf.set(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
ourUrl);
CredentialProviderHelper.createCredentialEntry(conf, alias, pw);

21
src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java
View File

@ -32,6 +32,7 @@
import org.apache.sqoop.testutil.S3CredentialGenerator;
import org.apache.sqoop.testutil.S3TestUtils;
import org.apache.sqoop.testutil.TextFileTestUtils;
import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory;
import org.apache.sqoop.util.password.CredentialProviderHelper;
import org.junit.After;
import org.junit.AfterClass;
@ -43,18 +44,36 @@
import org.junit.contrib.java.lang.system.EnvironmentVariables;
import org.junit.experimental.categories.Category;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays;
import static junit.framework.TestCase.fail;
@Category(S3Test.class)
@RunWith(Parameterized.class)
@Parameterized.UseParametersRunnerFactory(BlockJUnit4ClassRunnerWithParametersFactory.class)
public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase {
@Parameterized.Parameters(name = "credentialProviderPathProperty = {0}")
public static Iterable<? extends Object> parameters() {
return Arrays.asList(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
CredentialProviderHelper.S3A_CREDENTIAL_PROVIDER_PATH);
}
public static final Log LOG = LogFactory.getLog(
TestS3ImportWithHadoopCredProvider.class.getName());
private String credentialProviderPathProperty;
public TestS3ImportWithHadoopCredProvider(String credentialProviderPathProperty) {
this.credentialProviderPathProperty = credentialProviderPathProperty;
}
private static S3CredentialGenerator s3CredentialGenerator;
private static String providerPathDefault;
@ -156,7 +175,7 @@ public void testCredentialProviderWithNoPwdFileFails() throws Exception {
private String[] getArgs(String providerPath, boolean withPwdFile, String pwdFile) {
ArgumentArrayBuilder builder = S3TestUtils.getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(this);
builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, providerPath);
builder.withProperty(credentialProviderPathProperty, providerPath);
if (withPwdFile) {
builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, pwdFile);
}