github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-02 07:21:58 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3407 Introduce methods instead of TEMP_BASE_DIR and LOCAL_WAREHOUSE_DIR static fields

Browse Source 
(Szabolcs Vasas via Fero Szabo)
This commit is contained in:
Fero Szabo 2018-11-21 13:20:17 +01:00
parent bb9c2dd85b
commit 28896c8df1
8 changed files with 52 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/test/org/apache/sqoop/TestIncrementalImport.java
View File

@ -280,7 +280,7 @@ private void createIdVarcharTable(String tableName,
}
private Path getTablePath(String tableName) {
Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir());
return new Path(warehouse, tableName);
}
@ -303,7 +303,7 @@ public void clearDir(String tableName) {
public void createDir(String tableName) throws IOException {
FileSystem fs = FileSystem.getLocal(new Configuration());
Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir());
Path tableDir = new Path(warehouse, tableName);
fs.mkdirs(tableDir);
}
@ -316,7 +316,7 @@ public void createDir(String tableName) throws IOException {
public void assertDirOfNumbers(String tableName, int expectedNums) {
try {
FileSystem fs = FileSystem.getLocal(new Configuration());
Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir());
Path tableDir = new Path(warehouse, tableName);
FileStatus [] stats = fs.listStatus(tableDir);
String [] fileNames = new String[stats.length];
@ -368,7 +368,7 @@ public void assertDirOfNumbers(String tableName, int expectedNums) {
public void assertDirOfNumbersAndTimestamps(String tableName, int expectedNums) {
try {
FileSystem fs = FileSystem.getLocal(new Configuration());
Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir());
Path tableDir = new Path(warehouse, tableName);
FileStatus [] stats = fs.listStatus(tableDir);
String [] fileNames = new String[stats.length];
@ -419,7 +419,7 @@ public void assertDirOfNumbersAndTimestamps(String tableName, int expectedNums)
public void assertFirstSpecificNumber(String tableName, int val) {
try {
FileSystem fs = FileSystem.getLocal(new Configuration());
Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir());
Path tableDir = new Path(warehouse, tableName);
FileStatus [] stats = fs.listStatus(tableDir);
String [] filePaths = new String[stats.length];
@ -472,7 +472,7 @@ public void assertFirstSpecificNumber(String tableName, int val) {
public void assertSpecificNumber(String tableName, int val) {
try {
FileSystem fs = FileSystem.getLocal(new Configuration());
Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
Path warehouse = new Path(BaseSqoopTestCase.getLocalWarehouseDir());
Path tableDir = new Path(warehouse, tableName);
FileStatus [] stats = fs.listStatus(tableDir);
String [] filePaths = new String[stats.length];
@ -546,7 +546,7 @@ private List<String> getArgListForTable(String tableName, boolean commonArgs,
args.add("--table");
args.add(tableName);
args.add("--warehouse-dir");
args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
args.add(BaseSqoopTestCase.getLocalWarehouseDir());
if (isAppend) {
args.add("--incremental");
args.add("append");
@ -592,7 +592,7 @@ private List<String> getArgListForQuery(String query, String directoryName,
args.add("--class-name");
args.add(className);
args.add("--target-dir");
args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR
args.add(BaseSqoopTestCase.getLocalWarehouseDir()
+ System.getProperty("file.separator") + directoryName);
if (isAppend) {
args.add("--incremental");
@ -1322,7 +1322,7 @@ public void testIncrementalHiveAppendEmptyThenFull() throws Exception {
args.add("--table");
args.add(TABLE_NAME);
args.add("--warehouse-dir");
args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
args.add(BaseSqoopTestCase.getLocalWarehouseDir());
args.add("--hive-import");
args.add("--hive-table");
args.add(TABLE_NAME + "hive");

6
src/test/org/apache/sqoop/TestMerge.java
View File

@ -204,7 +204,7 @@ public void runMergeTest(SqoopOptions.FileLayout fileLayout) throws Exception {
// Now merge the results!
ClassLoaderStack.addJarFile(jarFileName, MERGE_CLASS_NAME);
Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
Path warehouse = new Path(getWarehouseDir());
options = getSqoopOptions(newConf());
options.setMergeOldPath(new Path(warehouse, OLD_PATH).toString());
options.setMergeNewPath(new Path(warehouse, NEW_PATH).toString());
@ -243,7 +243,7 @@ private void importData(String targetDir, SqoopOptions.FileLayout fileLayout) {
options.setFileLayout(fileLayout);
options.setDeleteMode(true);
Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
Path warehouse = new Path(getWarehouseDir());
options.setTargetDir(new Path(warehouse, targetDir).toString());
ImportTool importTool = new ImportTool();
@ -331,7 +331,7 @@ protected boolean checkFileForLine(FileSystem fs, Path p, SqoopOptions.FileLayou
protected boolean recordStartsWith(List<Integer> record, String dirName,
SqoopOptions.FileLayout fileLayout)
throws Exception {
Path warehousePath = new Path(LOCAL_WAREHOUSE_DIR);
Path warehousePath = new Path(getWarehouseDir());
Path targetPath = new Path(warehousePath, dirName);
FileSystem fs = FileSystem.getLocal(new Configuration());

18
src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java
View File

@ -74,7 +74,7 @@ public void setUp() {
@Test
public void testPasswordFilePathInOptionIsEnabled() throws Exception {
String passwordFilePath = TEMP_BASE_DIR + ".pwd";
String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd";
createTempFile(passwordFilePath);
try {
@ -98,7 +98,7 @@ public void testPasswordFileDoesNotExist() throws Exception {
try {
ArrayList<String> extraArgs = new ArrayList<String>();
extraArgs.add("--password-file");
extraArgs.add(TEMP_BASE_DIR + "unknown");
extraArgs.add(BaseSqoopTestCase.getTempBaseDir() + "unknown");
String[] argv = getCommonArgs(false, extraArgs);
Configuration conf = getConf();
@ -117,7 +117,7 @@ public void testPasswordFileIsADirectory() throws Exception {
try {
ArrayList<String> extraArgs = new ArrayList<String>();
extraArgs.add("--password-file");
extraArgs.add(TEMP_BASE_DIR);
extraArgs.add(BaseSqoopTestCase.getTempBaseDir());
String[] argv = getCommonArgs(false, extraArgs);
Configuration conf = getConf();
@ -133,7 +133,7 @@ public void testPasswordFileIsADirectory() throws Exception {
@Test
public void testBothPasswordOptions() throws Exception {
String passwordFilePath = TEMP_BASE_DIR + ".pwd";
String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd";
createTempFile(passwordFilePath);
try {
@ -162,7 +162,7 @@ public void testBothPasswordOptions() throws Exception {
@Test
public void testPasswordFilePath() throws Exception {
String passwordFilePath = TEMP_BASE_DIR + ".pwd";
String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd";
createTempFile(passwordFilePath);
writeToFile(passwordFilePath, "password");
@ -218,7 +218,7 @@ public void testPasswordNotInJobConf() throws Exception {
@Test
public void testPasswordInMetastoreWithRecordEnabledAndSecureOption()
throws Exception {
String passwordFilePath = TEMP_BASE_DIR + ".pwd";
String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd";
createTempFile(passwordFilePath);
ArrayList<String> extraArgs = new ArrayList<String>();
@ -255,7 +255,7 @@ public void testPasswordInMetastoreWithRecordEnabledAndSecureOption()
@Test
public void testPasswordInMetastoreWithRecordDisabledAndSecureOption()
throws Exception {
String passwordFilePath = TEMP_BASE_DIR + ".pwd";
String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd";
createTempFile(passwordFilePath);
ArrayList<String> extraArgs = new ArrayList<String>();
@ -386,7 +386,7 @@ public void testCredentialProviderLoader() throws Exception {
String alias = "super.secret.alias";
String pw = "super.secret.password";
String passwordFilePath = TEMP_BASE_DIR + ".pwd";
String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd";
String jksFile = "creds.jks";
createTempFile(passwordFilePath);
writeToFile(passwordFilePath, alias.getBytes());
@ -460,7 +460,7 @@ public void testPasswordAliasOption() throws Exception {
public void executeCipherTest(String password, String passphrase, String cipher, int keySize) throws Exception {
LOG.info("Using cipher: " + cipher + " with keySize " + keySize + " and passphrase " + passphrase );
String passwordFilePath = TEMP_BASE_DIR + ".pwd";
String passwordFilePath = BaseSqoopTestCase.getTempBaseDir() + ".pwd";
createTempFile(passwordFilePath);
writeToFile(passwordFilePath, encryptPassword(password, passphrase, cipher, 10000, keySize));
LOG.info("Generated encrypted password file in: " + passwordFilePath);

5
src/test/org/apache/sqoop/hbase/HBaseImportAddRowKeyTest.java
View File

@ -19,6 +19,7 @@
package org.apache.sqoop.hbase;
import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.testutil.BaseSqoopTestCase;
import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory;
import org.junit.Before;
import org.junit.Test;
@ -67,8 +68,8 @@ public void setUp() {
columnValues = new String[] { "0", "1" };
hbaseTableName = "addRowKeyTable";
hbaseColumnFamily = "addRowKeyFamily";
hbaseTmpDir = TEMP_BASE_DIR + "hbaseTmpDir";
hbaseBulkLoadDir = TEMP_BASE_DIR + "hbaseBulkLoadDir";
hbaseTmpDir = BaseSqoopTestCase.getTempBaseDir() + "hbaseTmpDir";
hbaseBulkLoadDir = BaseSqoopTestCase.getTempBaseDir() + "hbaseBulkLoadDir";
createTableWithColTypes(columnTypes, columnValues);
}

6
src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java
View File

@ -36,7 +36,7 @@
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.sqoop.testcategories.sqooptest.UnitTest;
import org.apache.sqoop.testutil.ImportJobTestCase;
import org.apache.sqoop.testutil.BaseSqoopTestCase;
import org.junit.Before;
import org.junit.Test;
@ -59,12 +59,12 @@ public class TestSplittableBufferedWriter {
TestSplittableBufferedWriter.class.getName());
private String getWriteDir() {
return new File(ImportJobTestCase.TEMP_BASE_DIR,
return new File(BaseSqoopTestCase.getTempBaseDir(),
"bufferedWriterTest").toString();
}
private Path getWritePath() {
return new Path(ImportJobTestCase.TEMP_BASE_DIR, "bufferedWriterTest");
return new Path(BaseSqoopTestCase.getTempBaseDir(), "bufferedWriterTest");
}
/** Create the directory where we'll write our test files to; and

7
src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java
View File

@ -28,6 +28,7 @@
import org.apache.sqoop.manager.SQLServerManager;
import org.apache.sqoop.testcategories.thirdpartytest.SqlServerTest;
import org.apache.sqoop.testutil.ArgumentArrayBuilder;
import org.apache.sqoop.testutil.BaseSqoopTestCase;
import org.apache.sqoop.testutil.ImportJobTestCase;
import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory;
import org.apache.sqoop.util.ExpectedLogMessage;
@ -326,7 +327,7 @@ private static ArgumentArrayBuilder getArgsBuilder() {
private static ArgumentArrayBuilder getArgsBuilderForTableImport() {
ArgumentArrayBuilder builder = getArgsBuilder();
return builder.withCommonHadoopFlags(true)
.withOption("warehouse-dir", LOCAL_WAREHOUSE_DIR)
.withOption("warehouse-dir", BaseSqoopTestCase.getLocalWarehouseDir())
.withOption("table", DBO_TABLE_NAME);
}
@ -334,13 +335,13 @@ private static ArgumentArrayBuilder getArgsBuilderForQueryImport() {
ArgumentArrayBuilder builder = getArgsBuilder();
return builder.withCommonHadoopFlags(true)
.withOption("query", "SELECT * FROM EMPLOYEES_MSSQL WHERE $CONDITIONS")
.withOption("target-dir", LOCAL_WAREHOUSE_DIR + "/" + DBO_TABLE_NAME);
.withOption("target-dir", BaseSqoopTestCase.getLocalWarehouseDir() + "/" + DBO_TABLE_NAME);
}
private static ArgumentArrayBuilder getArgsBuilderForDifferentSchemaTableImport() {
ArgumentArrayBuilder builder = getArgsBuilder();
return builder.withCommonHadoopFlags(true)
.withOption("warehouse-dir", LOCAL_WAREHOUSE_DIR)
.withOption("warehouse-dir", BaseSqoopTestCase.getLocalWarehouseDir())
.withOption("table", SCH_TABLE_NAME)
.withToolOption("schema", SCHEMA_SCH);
}

10
src/test/org/apache/sqoop/orm/TestClassWriter.java
View File

@ -35,6 +35,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.Shell;
import org.apache.sqoop.testcategories.sqooptest.IntegrationTest;
import org.apache.sqoop.testutil.BaseSqoopTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@ -45,7 +46,6 @@
import org.apache.sqoop.manager.ConnManager;
import org.apache.sqoop.testutil.DirUtil;
import org.apache.sqoop.testutil.HsqldbTestServer;
import org.apache.sqoop.testutil.ImportJobTestCase;
import org.apache.sqoop.tool.ImportTool;
import org.apache.sqoop.util.ClassLoaderStack;
import org.junit.experimental.categories.Category;
@ -100,9 +100,9 @@ public void setUp() {
// sanity check: make sure we're in a tmp dir before we blow anything away.
assertTrue("Test generates code in non-tmp dir!",
CODE_GEN_DIR.startsWith(ImportJobTestCase.TEMP_BASE_DIR));
CODE_GEN_DIR.startsWith(BaseSqoopTestCase.getTempBaseDir()));
assertTrue("Test generates jars in non-tmp dir!",
JAR_GEN_DIR.startsWith(ImportJobTestCase.TEMP_BASE_DIR));
JAR_GEN_DIR.startsWith(BaseSqoopTestCase.getTempBaseDir()));
// start out by removing these directories ahead of time
// to ensure that this is truly generating the code.
@ -134,9 +134,9 @@ public void tearDown() {
}
}
static final String CODE_GEN_DIR = ImportJobTestCase.TEMP_BASE_DIR
static final String CODE_GEN_DIR = BaseSqoopTestCase.getTempBaseDir()
+ "sqoop/test/codegen";
static final String JAR_GEN_DIR = ImportJobTestCase.TEMP_BASE_DIR
static final String JAR_GEN_DIR = BaseSqoopTestCase.getTempBaseDir()
+ "sqoop/test/jargen";
private File runGenerationTest(String[] argv, String classNameToCheck) {

34
src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java
View File

@ -88,23 +88,6 @@ private static void setOnPhysicalCluster(boolean value) {
public static final String MAP_OUTPUT_FILE_00001 = "part-m-00001";
public static final String REDUCE_OUTPUT_FILE_00000 = "part-r-00000";
/** Base directory for all temporary data. */
public static final String TEMP_BASE_DIR;
/** Where to import table data to in the local filesystem for testing. */
public static final String LOCAL_WAREHOUSE_DIR;
// Initializer for the above
static {
String tmpDir = System.getProperty("test.build.data", "/tmp/");
if (!tmpDir.endsWith(File.separator)) {
tmpDir = tmpDir + File.separator;
}
TEMP_BASE_DIR = tmpDir;
LOCAL_WAREHOUSE_DIR = TEMP_BASE_DIR + "sqoop/warehouse";
}
// Used if a test manually sets the table name to be used.
private String curTableName;
@ -136,7 +119,7 @@ protected String getTableName() {
}
protected String getWarehouseDir() {
return LOCAL_WAREHOUSE_DIR;
return getLocalWarehouseDir();
}
private String [] colNames;
@ -722,4 +705,19 @@ protected void resetDefaultTimeZone() {
TimeZone defaultTimeZone = TimeZone.getTimeZone(timeZoneId);
TimeZone.setDefault(defaultTimeZone);
}
/** Base directory for all temporary data. */
public static String getTempBaseDir() {
String tmpDir = System.getProperty("test.build.data", "/tmp/");
if (!tmpDir.endsWith(File.separator)) {
tmpDir = tmpDir + File.separator;
}
return tmpDir;
}
/** Where to import table data to in the local filesystem for testing. */
public static String getLocalWarehouseDir() {
return getTempBaseDir() + "sqoop/warehouse";
}
}