From 28896c8df1d151da29b3697c08fe5e81da4f3e94 Mon Sep 17 00:00:00 2001 From: Fero Szabo Date: Wed, 21 Nov 2018 13:20:17 +0100 Subject: [PATCH] SQOOP-3407 Introduce methods instead of TEMP_BASE_DIR and LOCAL_WAREHOUSE_DIR static fields (Szabolcs Vasas via Fero Szabo) --- .../apache/sqoop/TestIncrementalImport.java | 18 +++++----- src/test/org/apache/sqoop/TestMerge.java | 6 ++-- .../TestPassingSecurePassword.java | 18 +++++----- .../sqoop/hbase/HBaseImportAddRowKeyTest.java | 5 +-- .../io/TestSplittableBufferedWriter.java | 6 ++-- .../sqlserver/SQLServerManagerImportTest.java | 7 ++-- .../org/apache/sqoop/orm/TestClassWriter.java | 10 +++--- .../sqoop/testutil/BaseSqoopTestCase.java | 34 +++++++++---------- 8 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/test/org/apache/sqoop/TestIncrementalImport.java b/src/test/org/apache/sqoop/TestIncrementalImport.java index dbdd05c1..a1516adc 100644 --- a/src/test/org/apache/sqoop/TestIncrementalImport.java +++ b/src/test/org/apache/sqoop/TestIncrementalImport.java @@ -280,7 +280,7 @@ private void createIdVarcharTable(String tableName, } private Path getTablePath(String tableName) { - Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir()); return new Path(warehouse, tableName); } @@ -303,7 +303,7 @@ public void clearDir(String tableName) { public void createDir(String tableName) throws IOException { FileSystem fs = FileSystem.getLocal(new Configuration()); - Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir()); Path tableDir = new Path(warehouse, tableName); fs.mkdirs(tableDir); } @@ -316,7 +316,7 @@ public void createDir(String tableName) throws IOException { public void assertDirOfNumbers(String tableName, int expectedNums) { try { FileSystem fs = FileSystem.getLocal(new Configuration()); - Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir()); Path tableDir = new Path(warehouse, tableName); FileStatus [] stats = fs.listStatus(tableDir); String [] fileNames = new String[stats.length]; @@ -368,7 +368,7 @@ public void assertDirOfNumbers(String tableName, int expectedNums) { public void assertDirOfNumbersAndTimestamps(String tableName, int expectedNums) { try { FileSystem fs = FileSystem.getLocal(new Configuration()); - Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir()); Path tableDir = new Path(warehouse, tableName); FileStatus [] stats = fs.listStatus(tableDir); String [] fileNames = new String[stats.length]; @@ -419,7 +419,7 @@ public void assertDirOfNumbersAndTimestamps(String tableName, int expectedNums) public void assertFirstSpecificNumber(String tableName, int val) { try { FileSystem fs = FileSystem.getLocal(new Configuration()); - Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir()); Path tableDir = new Path(warehouse, tableName); FileStatus [] stats = fs.listStatus(tableDir); String [] filePaths = new String[stats.length]; @@ -472,7 +472,7 @@ public void assertFirstSpecificNumber(String tableName, int val) { public void assertSpecificNumber(String tableName, int val) { try { FileSystem fs = FileSystem.getLocal(new Configuration()); - Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir()); Path tableDir = new Path(warehouse, tableName); FileStatus [] stats = fs.listStatus(tableDir); String [] filePaths = new String[stats.length]; @@ -546,7 +546,7 @@ private List getArgListForTable(String tableName, boolean commonArgs, args.add("--table"); args.add(tableName); args.add("--warehouse-dir"); - args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + args.add(BaseSqoopTestCase.getLocalWarehouseDir()); if (isAppend) { args.add("--incremental"); args.add("append"); @@ -592,7 +592,7 @@ private List getArgListForQuery(String query, String directoryName, args.add("--class-name"); args.add(className); args.add("--target-dir"); - args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR + args.add(BaseSqoopTestCase.getLocalWarehouseDir() + System.getProperty("file.separator") + directoryName); if (isAppend) { args.add("--incremental"); @@ -1322,7 +1322,7 @@ public void testIncrementalHiveAppendEmptyThenFull() throws Exception { args.add("--table"); args.add(TABLE_NAME); args.add("--warehouse-dir"); - args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + args.add(BaseSqoopTestCase.getLocalWarehouseDir()); args.add("--hive-import"); args.add("--hive-table"); args.add(TABLE_NAME + "hive"); diff --git a/src/test/org/apache/sqoop/TestMerge.java b/src/test/org/apache/sqoop/TestMerge.java index b283174b..72632d72 100644 --- a/src/test/org/apache/sqoop/TestMerge.java +++ b/src/test/org/apache/sqoop/TestMerge.java @@ -204,7 +204,7 @@ public void runMergeTest(SqoopOptions.FileLayout fileLayout) throws Exception { // Now merge the results! ClassLoaderStack.addJarFile(jarFileName, MERGE_CLASS_NAME); - Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + Path warehouse = new Path(getWarehouseDir()); options = getSqoopOptions(newConf()); options.setMergeOldPath(new Path(warehouse, OLD_PATH).toString()); options.setMergeNewPath(new Path(warehouse, NEW_PATH).toString()); @@ -243,7 +243,7 @@ private void importData(String targetDir, SqoopOptions.FileLayout fileLayout) { options.setFileLayout(fileLayout); options.setDeleteMode(true); - Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + Path warehouse = new Path(getWarehouseDir()); options.setTargetDir(new Path(warehouse, targetDir).toString()); ImportTool importTool = new ImportTool(); @@ -331,7 +331,7 @@ protected boolean checkFileForLine(FileSystem fs, Path p, SqoopOptions.FileLayou protected boolean recordStartsWith(List record, String dirName, SqoopOptions.FileLayout fileLayout) throws Exception { - Path warehousePath = new Path(LOCAL_WAREHOUSE_DIR); + Path warehousePath = new Path(getWarehouseDir()); Path targetPath = new Path(warehousePath, dirName); FileSystem fs = FileSystem.getLocal(new Configuration()); diff --git a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java index 9c1e9f9a..dca3195b 100644 --- a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java +++ b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java @@ -74,7 +74,7 @@ public void setUp() { @Test public void testPasswordFilePathInOptionIsEnabled() throws Exception { - String passwordFilePath = TEMP_BASE_DIR + ".pwd"; + String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd"; createTempFile(passwordFilePath); try { @@ -98,7 +98,7 @@ public void testPasswordFileDoesNotExist() throws Exception { try { ArrayList extraArgs = new ArrayList(); extraArgs.add("--password-file"); - extraArgs.add(TEMP_BASE_DIR + "unknown"); + extraArgs.add(BaseSqoopTestCase.getTempBaseDir() + "unknown"); String[] argv = getCommonArgs(false, extraArgs); Configuration conf = getConf(); @@ -117,7 +117,7 @@ public void testPasswordFileIsADirectory() throws Exception { try { ArrayList extraArgs = new ArrayList(); extraArgs.add("--password-file"); - extraArgs.add(TEMP_BASE_DIR); + extraArgs.add(BaseSqoopTestCase.getTempBaseDir()); String[] argv = getCommonArgs(false, extraArgs); Configuration conf = getConf(); @@ -133,7 +133,7 @@ public void testPasswordFileIsADirectory() throws Exception { @Test public void testBothPasswordOptions() throws Exception { - String passwordFilePath = TEMP_BASE_DIR + ".pwd"; + String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd"; createTempFile(passwordFilePath); try { @@ -162,7 +162,7 @@ public void testBothPasswordOptions() throws Exception { @Test public void testPasswordFilePath() throws Exception { - String passwordFilePath = TEMP_BASE_DIR + ".pwd"; + String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd"; createTempFile(passwordFilePath); writeToFile(passwordFilePath, "password"); @@ -218,7 +218,7 @@ public void testPasswordNotInJobConf() throws Exception { @Test public void testPasswordInMetastoreWithRecordEnabledAndSecureOption() throws Exception { - String passwordFilePath = TEMP_BASE_DIR + ".pwd"; + String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd"; createTempFile(passwordFilePath); ArrayList extraArgs = new ArrayList(); @@ -255,7 +255,7 @@ public void testPasswordInMetastoreWithRecordEnabledAndSecureOption() @Test public void testPasswordInMetastoreWithRecordDisabledAndSecureOption() throws Exception { - String passwordFilePath = TEMP_BASE_DIR + ".pwd"; + String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd"; createTempFile(passwordFilePath); ArrayList extraArgs = new ArrayList(); @@ -386,7 +386,7 @@ public void testCredentialProviderLoader() throws Exception { String alias = "super.secret.alias"; String pw = "super.secret.password"; - String passwordFilePath = TEMP_BASE_DIR + ".pwd"; + String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd"; String jksFile = "creds.jks"; createTempFile(passwordFilePath); writeToFile(passwordFilePath, alias.getBytes()); @@ -460,7 +460,7 @@ public void testPasswordAliasOption() throws Exception { public void executeCipherTest(String password, String passphrase, String cipher, int keySize) throws Exception { LOG.info("Using cipher: " + cipher + " with keySize " + keySize + " and passphrase " + passphrase ); - String passwordFilePath = TEMP_BASE_DIR + ".pwd"; + String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd"; createTempFile(passwordFilePath); writeToFile(passwordFilePath, encryptPassword(password, passphrase, cipher, 10000, keySize)); LOG.info("Generated encrypted password file in: " + passwordFilePath); diff --git a/src/test/org/apache/sqoop/hbase/HBaseImportAddRowKeyTest.java b/src/test/org/apache/sqoop/hbase/HBaseImportAddRowKeyTest.java index df1840b3..be9bbd0b 100644 --- a/src/test/org/apache/sqoop/hbase/HBaseImportAddRowKeyTest.java +++ b/src/test/org/apache/sqoop/hbase/HBaseImportAddRowKeyTest.java @@ -19,6 +19,7 @@ package org.apache.sqoop.hbase; import org.apache.commons.lang.StringUtils; +import org.apache.sqoop.testutil.BaseSqoopTestCase; import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory; import org.junit.Before; import org.junit.Test; @@ -67,8 +68,8 @@ public void setUp() { columnValues = new String[] { "0", "1" }; hbaseTableName = "addRowKeyTable"; hbaseColumnFamily = "addRowKeyFamily"; - hbaseTmpDir = TEMP_BASE_DIR + "hbaseTmpDir"; - hbaseBulkLoadDir = TEMP_BASE_DIR + "hbaseBulkLoadDir"; + hbaseTmpDir = BaseSqoopTestCase.getTempBaseDir() + "hbaseTmpDir"; + hbaseBulkLoadDir = BaseSqoopTestCase.getTempBaseDir() + "hbaseBulkLoadDir"; createTableWithColTypes(columnTypes, columnValues); } diff --git a/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java b/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java index 71d69714..af4de138 100644 --- a/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java +++ b/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java @@ -36,7 +36,7 @@ import org.apache.hadoop.io.compress.GzipCodec; import org.apache.sqoop.testcategories.sqooptest.UnitTest; -import org.apache.sqoop.testutil.ImportJobTestCase; +import org.apache.sqoop.testutil.BaseSqoopTestCase; import org.junit.Before; import org.junit.Test; @@ -59,12 +59,12 @@ public class TestSplittableBufferedWriter { TestSplittableBufferedWriter.class.getName()); private String getWriteDir() { - return new File(ImportJobTestCase.TEMP_BASE_DIR, + return new File(BaseSqoopTestCase.getTempBaseDir(), "bufferedWriterTest").toString(); } private Path getWritePath() { - return new Path(ImportJobTestCase.TEMP_BASE_DIR, "bufferedWriterTest"); + return new Path(BaseSqoopTestCase.getTempBaseDir(), "bufferedWriterTest"); } /** Create the directory where we'll write our test files to; and diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java index ea7942f6..3c5bb327 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java @@ -28,6 +28,7 @@ import org.apache.sqoop.manager.SQLServerManager; import org.apache.sqoop.testcategories.thirdpartytest.SqlServerTest; import org.apache.sqoop.testutil.ArgumentArrayBuilder; +import org.apache.sqoop.testutil.BaseSqoopTestCase; import org.apache.sqoop.testutil.ImportJobTestCase; import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory; import org.apache.sqoop.util.ExpectedLogMessage; @@ -326,7 +327,7 @@ private static ArgumentArrayBuilder getArgsBuilder() { private static ArgumentArrayBuilder getArgsBuilderForTableImport() { ArgumentArrayBuilder builder = getArgsBuilder(); return builder.withCommonHadoopFlags(true) - .withOption("warehouse-dir", LOCAL_WAREHOUSE_DIR) + .withOption("warehouse-dir", BaseSqoopTestCase.getLocalWarehouseDir()) .withOption("table", DBO_TABLE_NAME); } @@ -334,13 +335,13 @@ private static ArgumentArrayBuilder getArgsBuilderForQueryImport() { ArgumentArrayBuilder builder = getArgsBuilder(); return builder.withCommonHadoopFlags(true) .withOption("query", "SELECT * FROM EMPLOYEES_MSSQL WHERE $CONDITIONS") - .withOption("target-dir", LOCAL_WAREHOUSE_DIR + "/" + DBO_TABLE_NAME); + .withOption("target-dir", BaseSqoopTestCase.getLocalWarehouseDir() + "/" + DBO_TABLE_NAME); } private static ArgumentArrayBuilder getArgsBuilderForDifferentSchemaTableImport() { ArgumentArrayBuilder builder = getArgsBuilder(); return builder.withCommonHadoopFlags(true) - .withOption("warehouse-dir", LOCAL_WAREHOUSE_DIR) + .withOption("warehouse-dir", BaseSqoopTestCase.getLocalWarehouseDir()) .withOption("table", SCH_TABLE_NAME) .withToolOption("schema", SCHEMA_SCH); } diff --git a/src/test/org/apache/sqoop/orm/TestClassWriter.java b/src/test/org/apache/sqoop/orm/TestClassWriter.java index 59a8908f..61315d59 100644 --- a/src/test/org/apache/sqoop/orm/TestClassWriter.java +++ b/src/test/org/apache/sqoop/orm/TestClassWriter.java @@ -35,6 +35,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.util.Shell; import org.apache.sqoop.testcategories.sqooptest.IntegrationTest; +import org.apache.sqoop.testutil.BaseSqoopTestCase; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -45,7 +46,6 @@ import org.apache.sqoop.manager.ConnManager; import org.apache.sqoop.testutil.DirUtil; import org.apache.sqoop.testutil.HsqldbTestServer; -import org.apache.sqoop.testutil.ImportJobTestCase; import org.apache.sqoop.tool.ImportTool; import org.apache.sqoop.util.ClassLoaderStack; import org.junit.experimental.categories.Category; @@ -100,9 +100,9 @@ public void setUp() { // sanity check: make sure we're in a tmp dir before we blow anything away. assertTrue("Test generates code in non-tmp dir!", - CODE_GEN_DIR.startsWith(ImportJobTestCase.TEMP_BASE_DIR)); + CODE_GEN_DIR.startsWith(BaseSqoopTestCase.getTempBaseDir())); assertTrue("Test generates jars in non-tmp dir!", - JAR_GEN_DIR.startsWith(ImportJobTestCase.TEMP_BASE_DIR)); + JAR_GEN_DIR.startsWith(BaseSqoopTestCase.getTempBaseDir())); // start out by removing these directories ahead of time // to ensure that this is truly generating the code. @@ -134,9 +134,9 @@ public void tearDown() { } } - static final String CODE_GEN_DIR = ImportJobTestCase.TEMP_BASE_DIR + static final String CODE_GEN_DIR = BaseSqoopTestCase.getTempBaseDir() + "sqoop/test/codegen"; - static final String JAR_GEN_DIR = ImportJobTestCase.TEMP_BASE_DIR + static final String JAR_GEN_DIR = BaseSqoopTestCase.getTempBaseDir() + "sqoop/test/jargen"; private File runGenerationTest(String[] argv, String classNameToCheck) { diff --git a/src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java b/src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java index e23aad3e..04af9fd6 100644 --- a/src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java +++ b/src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java @@ -88,23 +88,6 @@ private static void setOnPhysicalCluster(boolean value) { public static final String MAP_OUTPUT_FILE_00001 = "part-m-00001"; public static final String REDUCE_OUTPUT_FILE_00000 = "part-r-00000"; - /** Base directory for all temporary data. */ - public static final String TEMP_BASE_DIR; - - /** Where to import table data to in the local filesystem for testing. */ - public static final String LOCAL_WAREHOUSE_DIR; - - // Initializer for the above - static { - String tmpDir = System.getProperty("test.build.data", "/tmp/"); - if (!tmpDir.endsWith(File.separator)) { - tmpDir = tmpDir + File.separator; - } - - TEMP_BASE_DIR = tmpDir; - LOCAL_WAREHOUSE_DIR = TEMP_BASE_DIR + "sqoop/warehouse"; - } - // Used if a test manually sets the table name to be used. private String curTableName; @@ -136,7 +119,7 @@ protected String getTableName() { } protected String getWarehouseDir() { - return LOCAL_WAREHOUSE_DIR; + return getLocalWarehouseDir(); } private String [] colNames; @@ -722,4 +705,19 @@ protected void resetDefaultTimeZone() { TimeZone defaultTimeZone = TimeZone.getTimeZone(timeZoneId); TimeZone.setDefault(defaultTimeZone); } + + /** Base directory for all temporary data. */ + public static String getTempBaseDir() { + String tmpDir = System.getProperty("test.build.data", "/tmp/"); + if (!tmpDir.endsWith(File.separator)) { + tmpDir = tmpDir + File.separator; + } + + return tmpDir; + } + + /** Where to import table data to in the local filesystem for testing. */ + public static String getLocalWarehouseDir() { + return getTempBaseDir() + "sqoop/warehouse"; + } }