github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-02 05:12:08 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3391: Test storing AWS credentials in Hadoop CredentialProvider during import

Browse Source 
(Boglarka Egyed)
This commit is contained in:
Boglarka Egyed 2018-10-18 16:08:13 +02:00
parent 5dd8c8aad1
commit 1f7b0bf841
10 changed files with 238 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
build.gradle
View File

@ -141,6 +141,7 @@ dependencies {
testCompile group: 'junit', name: 'junit', version: junitVersion
testCompile group: 'org.assertj', name: 'assertj-core', version: assertjVersion
testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoallVersion
testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: systemRulesVersion
testCompile group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion, ext: 'jar'
}

4
build.xml
View File

@ -745,6 +745,10 @@
<copy todir="${test.build.extraconf}/oraoop">
<fileset dir="${test.dir}/oraoop"/>
</copy>
<copy file="${basedir}/conf/password-file.txt"
todir="${test.build.extraconf}" />
<copy file="${basedir}/conf/wrong-password-file.txt"
todir="${test.build.extraconf}" />
<junit
printsummary="yes" showoutput="${test.output}"
haltonfailure="no" fork="yes" maxmemory="5120m"

1
conf/password-file.txt Normal file
View File

@ -0,0 +1 @@
credProviderPwd

1
conf/wrong-password-file.txt Normal file
View File

@ -0,0 +1 @@
wrongCredProviderPwd

1
gradle.properties
View File

@ -48,6 +48,7 @@ commonsnetVersion=3.1
log4jVersion=1.2.16
junitVersion=4.12
mockitoallVersion=1.9.5
systemRulesVersion=1.17.0
assertjVersion=2.8.0
checkstyleVersion=5.5

2
ivy.xml
View File

@ -105,6 +105,8 @@ under the License.
conf="test->default"/>
<dependency org="org.mockito" name="mockito-all"
rev="${mockito-all.version}" conf="test->default"/>
<dependency org="com.github.stefanbirkner" name="system-rules"
rev="${system-rules.version}" conf="test->default"/>
<!-- We're only using H2 for tests as it supports stored
procedures; once we move to HSQLDB 2.x we can drop
this -->

1
ivy/libraries.properties
View File

@ -41,6 +41,7 @@ ivy.version=2.3.0
junit.version=4.12
assertj.version=2.8.0
mockito-all.version=1.9.5
system-rules.version=1.17.0
h2.version=1.3.170

2
src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
View File

@ -85,6 +85,8 @@ public class CredentialProviderHelper {
// Should track what is in CredentialProvider class.
public static final String CREDENTIAL_PROVIDER_PATH =
"hadoop.security.credential.provider.path";
public static final String CREDENTIAL_PROVIDER_PASSWORD_FILE =
"hadoop.security.credstore.java-keystore-provider.password-file";
public static boolean isProviderAvailable() {

213
src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java Normal file
View File

@ -0,0 +1,213 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.s3;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.security.alias.CredentialShell;
import org.apache.hadoop.util.ToolRunner;
import org.apache.sqoop.testutil.ArgumentArrayBuilder;
import org.apache.sqoop.testutil.DefaultS3CredentialGenerator;
import org.apache.sqoop.testutil.ImportJobTestCase;
import org.apache.sqoop.testutil.S3CredentialGenerator;
import org.apache.sqoop.testutil.S3TestUtils;
import org.apache.sqoop.testutil.TextFileTestUtils;
import org.apache.sqoop.util.password.CredentialProviderHelper;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.EnvironmentVariables;
import org.junit.rules.ExpectedException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import static junit.framework.TestCase.fail;
public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog(
TestS3ImportWithHadoopCredProvider.class.getName());
private static S3CredentialGenerator s3CredentialGenerator;
private static String providerPathDefault;
private static String providerPathEnv;
private static String providerPathPwdFile;
@ClassRule
public static final EnvironmentVariables environmentVariables
= new EnvironmentVariables();
private static File providerFileDefault;
private static File providerFileEnvPwd;
private static File providerFilePwdFile;
private FileSystem s3Client;
private static final String PASSWORD_FILE_NAME = "password-file.txt";
private static final String HADOOP_CREDSTORE_PASSWORD_ENV_NAME = "HADOOP_CREDSTORE_PASSWORD";
@Rule
public ExpectedException thrown = ExpectedException.none();
@BeforeClass
public static void setupS3Credentials() throws Exception {
String generatorCommand = S3TestUtils.getGeneratorCommand();
if (generatorCommand != null) {
s3CredentialGenerator = new DefaultS3CredentialGenerator(generatorCommand);
}
generateTempProviderFileNames();
fillCredentialProviderDefault();
fillCredentialProviderPwdFile();
fillCredentialProviderEnv();
}
@Before
public void setup() throws IOException {
S3TestUtils.runTestCaseOnlyIfS3CredentialsAreSet(s3CredentialGenerator);
super.setUp();
S3TestUtils.createTestTableFromInputData(this);
s3Client = S3TestUtils.setupS3ImportTestCase(s3CredentialGenerator);
environmentVariables.clear(HADOOP_CREDSTORE_PASSWORD_ENV_NAME);
}
@After
public void cleanUpTargetDir() {
S3TestUtils.tearDownS3ImportTestCase(s3Client);
super.tearDown();
}
@AfterClass
public static void deleteTemporaryCredFiles() {
providerFileDefault.deleteOnExit();
providerFileEnvPwd.deleteOnExit();
providerFilePwdFile.deleteOnExit();
}
@Test
public void testCredentialProviderDefaultSucceeds() throws Exception {
runImport(getArgs(providerPathDefault,false, null));
TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath());
}
@Test
public void testCredentialProviderEnvSucceeds() throws Exception {
setHadoopCredStorePwdEnvVar();
runImport(getArgs(providerPathEnv,false, null));
TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath());
}
@Test
public void testCredentialProviderPwdFileSucceeds() throws Exception {
runImport(getArgs(providerPathPwdFile,true, PASSWORD_FILE_NAME));
TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath());
}
@Test
public void testCredentialProviderWithNoProviderPathFails() throws Exception {
thrown.expect(IOException.class);
runImport(getArgs(null,false, null));
}
@Test
public void testCredentialProviderWithNoEnvFails() throws Exception {
thrown.expect(IOException.class);
runImport(getArgs(providerPathEnv,false, null));
}
@Test
public void testCredentialProviderWithWrongPwdFileFails() throws Exception {
thrown.expect(IOException.class);
runImport(getArgs(providerPathPwdFile,true, "wrong-password-file.txt"));
}
@Test
public void testCredentialProviderWithNoPwdFileFails() throws Exception {
thrown.expect(IOException.class);
runImport(getArgs(providerPathPwdFile,true, null));
}
private String[] getArgs(String providerPath, boolean withPwdFile, String pwdFile) {
ArgumentArrayBuilder builder = S3TestUtils.getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(this);
builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, providerPath);
if (withPwdFile) {
builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, pwdFile);
}
return builder.build();
}
private static void fillCredentialProviderDefault() throws Exception {
fillCredentialProvider(new Configuration(), providerPathDefault);
}
private static void fillCredentialProviderEnv() throws Exception {
setHadoopCredStorePwdEnvVar();
fillCredentialProvider(new Configuration(), providerPathEnv);
}
private static void fillCredentialProviderPwdFile() throws Exception {
Configuration conf = new Configuration();
conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, PASSWORD_FILE_NAME);
fillCredentialProvider(conf, providerPathPwdFile);
}
private static void generateTempProviderFileNames() throws IOException {
providerFileDefault = Files.createTempFile("test-default-pwd-", ".jceks").toFile();
boolean deleted = providerFileDefault.delete();
providerFileEnvPwd = Files.createTempFile("test-env-pwd-", ".jceks").toFile();
deleted &= providerFileEnvPwd.delete();
providerFilePwdFile = Files.createTempFile("test-file-pwd-", ".jceks").toFile();
deleted &= providerFilePwdFile.delete();
if (!deleted) {
fail("Could not delete temporary provider files");
}
providerPathDefault = "jceks://file/" + providerFileDefault.getAbsolutePath();
providerPathEnv = "jceks://file/" + providerFileEnvPwd.getAbsolutePath();
providerPathPwdFile = "jceks://file/" + providerFilePwdFile.getAbsolutePath();
}
private static void runCredentialProviderCreateCommand(String command, Configuration conf) throws Exception {
ToolRunner.run(conf, new CredentialShell(), command.split(" "));
}
private static String getCreateCommand(String credentialKey, String credentialValue, String providerPath) {
return "create " + credentialKey + " -value " + credentialValue + " -provider " + providerPath;
}
private static void fillCredentialProvider(Configuration conf, String providerPath) throws Exception {
runCredentialProviderCreateCommand(getCreateCommand(Constants.ACCESS_KEY, s3CredentialGenerator.getS3AccessKey(), providerPath), conf);
runCredentialProviderCreateCommand(getCreateCommand(Constants.SECRET_KEY, s3CredentialGenerator.getS3SecretKey(), providerPath), conf);
if (s3CredentialGenerator.getS3SessionToken() != null) {
runCredentialProviderCreateCommand(getCreateCommand(Constants.SESSION_TOKEN, s3CredentialGenerator.getS3SessionToken(), providerPath), conf);
}
}
private static void setHadoopCredStorePwdEnvVar() {
environmentVariables.set(HADOOP_CREDSTORE_PASSWORD_ENV_NAME, "credProviderPwd");
}
}

12
src/test/org/apache/sqoop/testutil/S3TestUtils.java
View File

@ -178,6 +178,18 @@ public static ArgumentArrayBuilder getArgumentArrayBuilderForS3UnitTests(BaseSqo
.withOption("target-dir", getTargetDirPath().toString());
}
public static ArgumentArrayBuilder getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(BaseSqoopTestCase testCase) {
ArgumentArrayBuilder builder = new ArgumentArrayBuilder();
return builder.withCommonHadoopFlags()
.withProperty("fs.s3a.impl.disable.cache", "true")
.withProperty(Constants.AWS_CREDENTIALS_PROVIDER, getTemporaryCredentialsProviderClass())
.withOption("connect", testCase.getConnectString())
.withOption("num-mappers", "1")
.withOption("table", testCase.getTableName())
.withOption("target-dir", getTargetDirPath().toString());
}
public static ArgumentArrayBuilder getArgumentArrayBuilderForS3UnitTestsWithFileFormatOption(BaseSqoopTestCase testCase,
S3CredentialGenerator s3CredentialGenerator,
String fileFormat) {