diff --git a/build.gradle b/build.gradle index 7a0712e3..2340bce7 100644 --- a/build.gradle +++ b/build.gradle @@ -141,6 +141,7 @@ dependencies { testCompile group: 'junit', name: 'junit', version: junitVersion testCompile group: 'org.assertj', name: 'assertj-core', version: assertjVersion testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoallVersion + testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: systemRulesVersion testCompile group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion, ext: 'jar' } diff --git a/build.xml b/build.xml index f3975317..995a5130 100644 --- a/build.xml +++ b/build.xml @@ -745,6 +745,10 @@ + + + diff --git a/ivy/libraries.properties b/ivy/libraries.properties index 2ca95ee9..3511a6f3 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -41,6 +41,7 @@ ivy.version=2.3.0 junit.version=4.12 assertj.version=2.8.0 mockito-all.version=1.9.5 +system-rules.version=1.17.0 h2.version=1.3.170 diff --git a/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java b/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java index 1d6481a0..4e79f0ae 100644 --- a/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java +++ b/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java @@ -85,6 +85,8 @@ public class CredentialProviderHelper { // Should track what is in CredentialProvider class. public static final String CREDENTIAL_PROVIDER_PATH = "hadoop.security.credential.provider.path"; + public static final String CREDENTIAL_PROVIDER_PASSWORD_FILE = + "hadoop.security.credstore.java-keystore-provider.password-file"; public static boolean isProviderAvailable() { diff --git a/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java b/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java new file mode 100644 index 00000000..e03eb64e --- /dev/null +++ b/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.s3; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.s3a.Constants; +import org.apache.hadoop.security.alias.CredentialShell; +import org.apache.hadoop.util.ToolRunner; +import org.apache.sqoop.testutil.ArgumentArrayBuilder; +import org.apache.sqoop.testutil.DefaultS3CredentialGenerator; +import org.apache.sqoop.testutil.ImportJobTestCase; +import org.apache.sqoop.testutil.S3CredentialGenerator; +import org.apache.sqoop.testutil.S3TestUtils; +import org.apache.sqoop.testutil.TextFileTestUtils; +import org.apache.sqoop.util.password.CredentialProviderHelper; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.contrib.java.lang.system.EnvironmentVariables; +import org.junit.rules.ExpectedException; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +import static junit.framework.TestCase.fail; + +public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase { + public static final Log LOG = LogFactory.getLog( + TestS3ImportWithHadoopCredProvider.class.getName()); + + private static S3CredentialGenerator s3CredentialGenerator; + + private static String providerPathDefault; + private static String providerPathEnv; + private static String providerPathPwdFile; + + @ClassRule + public static final EnvironmentVariables environmentVariables + = new EnvironmentVariables(); + private static File providerFileDefault; + private static File providerFileEnvPwd; + private static File providerFilePwdFile; + + private FileSystem s3Client; + + private static final String PASSWORD_FILE_NAME = "password-file.txt"; + private static final String HADOOP_CREDSTORE_PASSWORD_ENV_NAME = "HADOOP_CREDSTORE_PASSWORD"; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @BeforeClass + public static void setupS3Credentials() throws Exception { + String generatorCommand = S3TestUtils.getGeneratorCommand(); + if (generatorCommand != null) { + s3CredentialGenerator = new DefaultS3CredentialGenerator(generatorCommand); + } + generateTempProviderFileNames(); + fillCredentialProviderDefault(); + fillCredentialProviderPwdFile(); + fillCredentialProviderEnv(); + } + + @Before + public void setup() throws IOException { + S3TestUtils.runTestCaseOnlyIfS3CredentialsAreSet(s3CredentialGenerator); + super.setUp(); + S3TestUtils.createTestTableFromInputData(this); + s3Client = S3TestUtils.setupS3ImportTestCase(s3CredentialGenerator); + environmentVariables.clear(HADOOP_CREDSTORE_PASSWORD_ENV_NAME); + } + + @After + public void cleanUpTargetDir() { + S3TestUtils.tearDownS3ImportTestCase(s3Client); + super.tearDown(); + } + + @AfterClass + public static void deleteTemporaryCredFiles() { + providerFileDefault.deleteOnExit(); + providerFileEnvPwd.deleteOnExit(); + providerFilePwdFile.deleteOnExit(); + } + + @Test + public void testCredentialProviderDefaultSucceeds() throws Exception { + runImport(getArgs(providerPathDefault,false, null)); + TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath()); + } + + @Test + public void testCredentialProviderEnvSucceeds() throws Exception { + setHadoopCredStorePwdEnvVar(); + runImport(getArgs(providerPathEnv,false, null)); + TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath()); + } + + @Test + public void testCredentialProviderPwdFileSucceeds() throws Exception { + runImport(getArgs(providerPathPwdFile,true, PASSWORD_FILE_NAME)); + TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath()); + } + + @Test + public void testCredentialProviderWithNoProviderPathFails() throws Exception { + thrown.expect(IOException.class); + runImport(getArgs(null,false, null)); + } + + @Test + public void testCredentialProviderWithNoEnvFails() throws Exception { + thrown.expect(IOException.class); + runImport(getArgs(providerPathEnv,false, null)); + } + + @Test + public void testCredentialProviderWithWrongPwdFileFails() throws Exception { + thrown.expect(IOException.class); + runImport(getArgs(providerPathPwdFile,true, "wrong-password-file.txt")); + } + + @Test + public void testCredentialProviderWithNoPwdFileFails() throws Exception { + thrown.expect(IOException.class); + runImport(getArgs(providerPathPwdFile,true, null)); + } + + private String[] getArgs(String providerPath, boolean withPwdFile, String pwdFile) { + ArgumentArrayBuilder builder = S3TestUtils.getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(this); + + builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, providerPath); + if (withPwdFile) { + builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, pwdFile); + } + return builder.build(); + } + + private static void fillCredentialProviderDefault() throws Exception { + fillCredentialProvider(new Configuration(), providerPathDefault); + } + + private static void fillCredentialProviderEnv() throws Exception { + setHadoopCredStorePwdEnvVar(); + fillCredentialProvider(new Configuration(), providerPathEnv); + } + + private static void fillCredentialProviderPwdFile() throws Exception { + Configuration conf = new Configuration(); + conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, PASSWORD_FILE_NAME); + fillCredentialProvider(conf, providerPathPwdFile); + } + + private static void generateTempProviderFileNames() throws IOException { + providerFileDefault = Files.createTempFile("test-default-pwd-", ".jceks").toFile(); + boolean deleted = providerFileDefault.delete(); + providerFileEnvPwd = Files.createTempFile("test-env-pwd-", ".jceks").toFile(); + deleted &= providerFileEnvPwd.delete(); + providerFilePwdFile = Files.createTempFile("test-file-pwd-", ".jceks").toFile(); + deleted &= providerFilePwdFile.delete(); + if (!deleted) { + fail("Could not delete temporary provider files"); + } + providerPathDefault = "jceks://file/" + providerFileDefault.getAbsolutePath(); + providerPathEnv = "jceks://file/" + providerFileEnvPwd.getAbsolutePath(); + providerPathPwdFile = "jceks://file/" + providerFilePwdFile.getAbsolutePath(); + } + + private static void runCredentialProviderCreateCommand(String command, Configuration conf) throws Exception { + ToolRunner.run(conf, new CredentialShell(), command.split(" ")); + } + + private static String getCreateCommand(String credentialKey, String credentialValue, String providerPath) { + return "create " + credentialKey + " -value " + credentialValue + " -provider " + providerPath; + } + + private static void fillCredentialProvider(Configuration conf, String providerPath) throws Exception { + runCredentialProviderCreateCommand(getCreateCommand(Constants.ACCESS_KEY, s3CredentialGenerator.getS3AccessKey(), providerPath), conf); + runCredentialProviderCreateCommand(getCreateCommand(Constants.SECRET_KEY, s3CredentialGenerator.getS3SecretKey(), providerPath), conf); + + if (s3CredentialGenerator.getS3SessionToken() != null) { + runCredentialProviderCreateCommand(getCreateCommand(Constants.SESSION_TOKEN, s3CredentialGenerator.getS3SessionToken(), providerPath), conf); + } + } + + private static void setHadoopCredStorePwdEnvVar() { + environmentVariables.set(HADOOP_CREDSTORE_PASSWORD_ENV_NAME, "credProviderPwd"); + } +} diff --git a/src/test/org/apache/sqoop/testutil/S3TestUtils.java b/src/test/org/apache/sqoop/testutil/S3TestUtils.java index c9d17bc7..97d53bba 100644 --- a/src/test/org/apache/sqoop/testutil/S3TestUtils.java +++ b/src/test/org/apache/sqoop/testutil/S3TestUtils.java @@ -178,6 +178,18 @@ public static ArgumentArrayBuilder getArgumentArrayBuilderForS3UnitTests(BaseSqo .withOption("target-dir", getTargetDirPath().toString()); } + public static ArgumentArrayBuilder getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(BaseSqoopTestCase testCase) { + + ArgumentArrayBuilder builder = new ArgumentArrayBuilder(); + return builder.withCommonHadoopFlags() + .withProperty("fs.s3a.impl.disable.cache", "true") + .withProperty(Constants.AWS_CREDENTIALS_PROVIDER, getTemporaryCredentialsProviderClass()) + .withOption("connect", testCase.getConnectString()) + .withOption("num-mappers", "1") + .withOption("table", testCase.getTableName()) + .withOption("target-dir", getTargetDirPath().toString()); + } + public static ArgumentArrayBuilder getArgumentArrayBuilderForS3UnitTestsWithFileFormatOption(BaseSqoopTestCase testCase, S3CredentialGenerator s3CredentialGenerator, String fileFormat) {