github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 01:13:29 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3226: ImportTest, ExportTest and TimestampDataTest fail because of column escaping problems

Browse Source 
(Szabolcs Vasas via Boglarka Egyed)
This commit is contained in:
Boglarka Egyed 2017-08-29 11:56:24 +02:00
parent 606eb5f581
commit caec28c1fa
4 changed files with 42 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/test/org/apache/sqoop/manager/oracle/ExportTest.java
View File

@ -31,6 +31,8 @@ public class ExportTest extends OraOopTestCase {
private static final ExportTest TEST_CASE = new ExportTest();
private static final boolean DISABLE_ORACLE_ESCAPING_FLAG = false;
@BeforeClass
public static void setUpHdfsData() throws Exception {
// Copy the TST_PRODUCT table into HDFS which can be used for the export
@ -40,14 +42,14 @@ public static void setUpHdfsData() throws Exception {
TEST_CASE.createTable("table_tst_product.xml");
int retCode =
TEST_CASE.runImport("TST_PRODUCT", TEST_CASE.getSqoopConf(), false);
TEST_CASE.runImport("TST_PRODUCT", TEST_CASE.getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
}
@Test
public void testProductExport() throws Exception {
int retCode =
TEST_CASE.runExportFromTemplateTable("TST_PRODUCT", "TST_PRODUCT_EXP", false);
TEST_CASE.runExportFromTemplateTable("TST_PRODUCT", "TST_PRODUCT_EXP", false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
}
@ -55,7 +57,7 @@ public void testProductExport() throws Exception {
public void testProductExportMixedCaseTableName() throws Exception {
int retCode =
TEST_CASE.runExportFromTemplateTable("TST_PRODUCT",
"\"\"T5+_Pr#duct_Exp\"\"", false);
"\"\"T5+_Pr#duct_Exp\"\"", false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
}

31
src/test/org/apache/sqoop/manager/oracle/ImportTest.java
View File

@ -19,7 +19,6 @@
package org.apache.sqoop.manager.oracle;
import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.SqoopOptions;
import org.apache.sqoop.manager.oracle.OraOopConstants.
OraOopOracleDataChunkMethod;
import org.junit.Test;
@ -31,13 +30,15 @@
*/
public class ImportTest extends OraOopTestCase {
private static final boolean DISABLE_ORACLE_ESCAPING_FLAG = false;
@Test
public void testProductImport() throws Exception {
setSqoopTargetDirectory(getSqoopTargetDirectory() + "tst_product");
createTable("table_tst_product.xml");
try {
int retCode = runImport("TST_PRODUCT", getSqoopConf(), false);
int retCode = runImport("TST_PRODUCT", getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -49,16 +50,14 @@ public void testProductImport() throws Exception {
@Test
public void testProductWithWhiteSpaceImport() throws Exception {
System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "false");
setSqoopTargetDirectory(getSqoopTargetDirectory() + "tst_product");
createTable("table_tst_product_with_white_space.xml");
try {
int retCode = runImport("TST_Pr OdUCT", getSqoopConf(), false);
int retCode = runImport("TST_Pr OdUCT", getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED);
cleanupFolders();
closeTestEnvConnection();
}
@ -70,7 +69,7 @@ public void testProductPartImport() throws Exception {
createTable("table_tst_product_part.xml");
try {
int retCode = runImport("TST_PRODUCT_PART", getSqoopConf(), false);
int retCode = runImport("TST_PRODUCT_PART", getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -89,7 +88,7 @@ public void testProductPartImportPartitionChunk() throws Exception {
OraOopConstants.OraOopOracleDataChunkMethod.PARTITION.toString());
try {
int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false);
int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -110,7 +109,7 @@ public void testProductPartImportSubset() throws Exception {
"tst_product_part_1,tst_product_part_2,\"tst_product_pa#rt_6\"");
try {
int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false);
int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -134,7 +133,7 @@ public void testProductPartImportSubsetPartitionChunk() throws Exception {
+"tst_product_part_3,\"tst_product_pa#rt_6\"");
try {
int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false);
int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -149,7 +148,7 @@ public void testProductSubPartImport() throws Exception {
createTable("table_tst_product_subpart.xml");
try {
int retCode = runImport("TST_PRODUCT_SUBPART", getSqoopConf(), false);
int retCode = runImport("TST_PRODUCT_SUBPART", getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -168,7 +167,7 @@ public void testProductSubPartImportPartitionChunk() throws Exception {
OraOopConstants.OraOopOracleDataChunkMethod.PARTITION.toString());
try {
int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false);
int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -191,7 +190,7 @@ public void testProductSubPartImportSubset() throws Exception {
+"TST_PRODUCT_PART_3,TST_PRODUCT_PART_4");
try {
int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false);
int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -212,7 +211,7 @@ public void testProductSubPartImportSubsetPartitionChunk() throws Exception {
"TST_PRODUCT_PART_1,TST_PRODUCT_PART_2,TST_PRODUCT_PART_3");
try {
int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false);
int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -233,7 +232,7 @@ public void testProductImportConsistentRead() throws Exception {
sqoopConf.setBoolean(OraOopConstants.ORAOOP_IMPORT_CONSISTENT_READ, true);
try {
int retCode = runImport("TST_PRODUCT", sqoopConf, false);
int retCode = runImport("TST_PRODUCT", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -248,7 +247,7 @@ public void testProductImportMixedCaseTableName() throws Exception {
createTable("table_tst_product_special_chars.xml");
try {
int retCode = runImport("\"\"T5+_Pr#duct\"\"", getSqoopConf(), false);
int retCode = runImport("\"\"T5+_Pr#duct\"\"", getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {
@ -267,7 +266,7 @@ public void testProductPartIotImport() throws Exception {
OraOopConstants.OraOopOracleDataChunkMethod.PARTITION.toString());
try {
int retCode = runImport("TST_PRODUCT_PART_IOT", sqoopConf, false);
int retCode = runImport("TST_PRODUCT_PART_IOT", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {

22
src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java
View File

@ -55,7 +55,7 @@ public abstract class OraOopTestCase {
private static final OraOopLog LOG = OraOopLogFactory.getLog(
OraOopTestCase.class.getName());
private static final boolean ESCAPING_DISABLED_DEFAULT = true;
private String sqoopGenLibDirectory = System.getProperty("user.dir")
+ "/target/tmp/lib";
private String sqoopGenSrcDirectory = System.getProperty("user.dir")
@ -206,6 +206,11 @@ protected Configuration getSqoopConf() {
protected int runImport(String tableName, Configuration sqoopConf,
boolean sequenceFile) {
return runImport(tableName, sqoopConf, sequenceFile, ESCAPING_DISABLED_DEFAULT);
}
protected int runImport(String tableName, Configuration sqoopConf,
boolean sequenceFile, boolean escapingDisabled) {
Logger rootLogger = Logger.getRootLogger();
StringWriter stringWriter = new StringWriter();
Layout layout = new PatternLayout("%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n");
@ -246,6 +251,9 @@ protected int runImport(String tableName, Configuration sqoopConf,
sqoopArgs.add("--outdir");
sqoopArgs.add(this.sqoopGenSrcDirectory);
sqoopArgs.add("--oracle-escaping-disabled");
sqoopArgs.add(Boolean.toString(escapingDisabled));
if (OracleUtils.NUM_MAPPERS != 0) {
sqoopArgs.add("--num-mappers");
sqoopArgs.add(Integer.toString(OracleUtils.NUM_MAPPERS));
@ -276,16 +284,21 @@ protected int runImport(String tableName, Configuration sqoopConf,
protected int runExportFromTemplateTable(String templateTableName,
String tableName, boolean isPartitoned) {
return runExportFromTemplateTable(templateTableName, tableName, isPartitoned, ESCAPING_DISABLED_DEFAULT);
}
protected int runExportFromTemplateTable(String templateTableName,
String tableName, boolean isPartitoned, boolean escapingDisabled) {
Map<String, String> stringConfigEntries = new HashMap<String, String>();
stringConfigEntries.put("oraoop.template.table", templateTableName);
Map<String, Boolean> booleanConfigEntries = new HashMap<String, Boolean>();
booleanConfigEntries.put("oraoop.partitioned", isPartitoned);
return runExport(tableName, new ArrayList<String>(), stringConfigEntries, booleanConfigEntries);
return runExport(tableName, new ArrayList<String>(), stringConfigEntries, booleanConfigEntries, escapingDisabled);
}
protected int runExport(String tableName, List<String> additionalArgs, Map<String, String> stringConfigEntries,
Map<String, Boolean> booleanConfigEntries) {
Map<String, Boolean> booleanConfigEntries, boolean escapingDisabled) {
List<String> sqoopArgs = new ArrayList<String>();
sqoopArgs.add("export");
@ -316,6 +329,9 @@ protected int runExport(String tableName, List<String> additionalArgs, Map<Strin
sqoopArgs.add("--outdir");
sqoopArgs.add(this.sqoopGenSrcDirectory);
sqoopArgs.add("--oracle-escaping-disabled");
sqoopArgs.add(Boolean.toString(escapingDisabled));
sqoopArgs.addAll(additionalArgs);
Configuration sqoopConf = getSqoopConf();

4
src/test/org/apache/sqoop/manager/oracle/TimestampDataTest.java
View File

@ -30,6 +30,8 @@
*/
public class TimestampDataTest extends OraOopTestCase {
private static final boolean DISABLE_ORACLE_ESCAPING_FLAG = false;
@Test
public void testProductImportTimezone() throws Exception {
setSqoopTargetDirectory(getSqoopTargetDirectory() + "tst_product_timezone");
@ -39,7 +41,7 @@ public void testProductImportTimezone() throws Exception {
sqoopConf.setBoolean(OraOopConstants.ORAOOP_MAP_TIMESTAMP_AS_STRING, false);
try {
int retCode = runImport("TST_PRODUCT", sqoopConf, false);
int retCode = runImport("TST_PRODUCT", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG);
assertEquals("Return code should be 0", 0, retCode);
} finally {