SQOOP-3367: Improve third party tests to be able to execute them in a single JVM

(Szabolcs Vasas via Boglarka Egyed)

src/java/org/apache/sqoop/hive/HiveImport.java

@@ -42,6 +42,8 @@
 import org.apache.sqoop.manager.ConnManager;
 import org.apache.sqoop.util.ExitSecurityException;
 
+import static org.apache.commons.lang3.StringUtils.defaultString;
+
 /**
  * Utility to import a table into the Hive metastore. Manages the connection
  * to Hive itself as well as orchestrating the use of the other classes in this
@@ -159,7 +161,8 @@ public void importTable(String inputTableName, String outputTableName,
     boolean debugMode = expectedScript != null;
     if (debugMode) {
       env.add("EXPECTED_SCRIPT=" + expectedScript);
-      env.add("TMPDIR=" + options.getTempDir());
+      String tmpDir = defaultString(options.getWarehouseDir(), "/tmp");
+      env.add("TMPDIR=" + tmpDir);
     }
 
     // generate the HQL statements to run.

src/test/org/apache/sqoop/manager/oracle/OraOopTypesTest.java

@@ -31,6 +31,7 @@ public class OraOopTypesTest extends OraOopTestCase {
   public void ensureTypesAfterExportMappedAsExpected() throws Exception {
     try {
       setSqoopTargetDirectory(getSqoopTargetDirectory() + "types_test");
+      setSqoopGenClassName(OraOopTypesTest.class.getSimpleName());
       String tempTableName = "ORACLE_DATATYPES_TEMPLATE";
       String tableName = "ORACLE_DATATYPES";
       createTableFromSQL("create table " + tempTableName + " ("

src/test/org/apache/sqoop/manager/oracle/SystemImportTest.java

@@ -86,6 +86,7 @@ public void importTest() throws Exception {
     // Generate test data in oracle
     setSqoopTargetDirectory(getSqoopTargetDirectory()
         + OracleUtils.SYSTEMTEST_TABLE_NAME);
+    setSqoopGenClassName(SystemImportTest.class.getSimpleName());
     int numRows = OracleUtils.SYSTEMTEST_NUM_ROWS;
     Connection conn = getTestEnvConnection();
     OraOopOracleQueries.setConnectionTimeZone(conn, "GMT");

src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java

@@ -45,6 +45,7 @@
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.List;
+import java.util.TimeZone;
 
 import static org.apache.commons.lang3.StringUtils.wrap;
 import static org.junit.Assert.fail;
@@ -217,6 +218,7 @@ protected SqoopOptions getSqoopOptions(Configuration conf) {
   public void setUp() {
     // The assumption is that correct HADOOP configuration will have it set to
     // hdfs://namenode
+    resetDefaultTimeZone();
     setOnPhysicalCluster(
         !CommonArgs.LOCAL_FS.equals(System.getProperty(
             CommonArgs.FS_DEFAULT_NAME)));
@@ -711,4 +713,10 @@ public static long timeFromString(String timeStampString) {
       throw new RuntimeException(e);
     }
   }
+
+  protected void resetDefaultTimeZone() {
+    String timeZoneId = System.getProperty("user.timezone");
+    TimeZone defaultTimeZone = TimeZone.getTimeZone(timeZoneId);
+    TimeZone.setDefault(defaultTimeZone);
+  }
 }

src/test/org/apache/sqoop/testutil/LobAvroImportTestCase.java

@@ -212,8 +212,7 @@ public void testBlobAvroImportExternal() throws IOException, SQLException {
     ByteBuffer buf = (ByteBuffer) record.get(getColName(0));
     String returnVal = new String(buf.array());
     String expectedStart = "externalLob(lf,_lob/large_obj";
-    String expectedEnd = getTableNum() + "_m_0000000.lob,68,"
+    String expectedEnd = "_m_0000000.lob,68," + data.length() + ")";
-      + data.length() + ")";
 
     assertNotNull(returnVal);
     assertTrue("ExpectedStart: " + expectedStart + ", value: " + returnVal, returnVal.startsWith(expectedStart));
@@ -303,8 +302,7 @@ public void testBlobCompressedAvroImportExternal()
     ByteBuffer buf = (ByteBuffer) record.get(getColName(0));
     String returnVal = new String(buf.array());
     String expectedStart = "externalLob(lf,_lob/large_obj";
-    String expectedEnd = getTableNum() + "_m_0000000.lob,68,"
+    String expectedEnd = "_m_0000000.lob,68," + data.length() + ")";
-      + data.length() + ")";
 
     assertNotNull(returnVal);
     assertTrue("ExpectedStart: " + expectedStart + ", value: " + returnVal, returnVal.startsWith(expectedStart));

testdata/hive/scripts/customDelimImport.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `CUSTOM_DELIM_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\054' LINES TERMINATED BY '\174' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/CUSTOM_DELIM_IMPORT' INTO TABLE `CUSTOM_DELIM_IMPORT`;
+LOAD DATA INPATH 'file:BASEPATH/CUSTOM_DELIM_IMPORT' INTO TABLE `CUSTOM_DELIM_IMPORT`;

testdata/hive/scripts/dateImport.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `DATE_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/DATE_HIVE_IMPORT' INTO TABLE `DATE_HIVE_IMPORT`;
+LOAD DATA INPATH 'file:BASEPATH/DATE_HIVE_IMPORT' INTO TABLE `DATE_HIVE_IMPORT`;

testdata/hive/scripts/decimalMapImport.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `DECIMAL_MAP_HIVE_IMPORT` ( `DATA_COL0` DECIMAL(10, 10), `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/DECIMAL_MAP_HIVE_IMPORT' INTO TABLE `DECIMAL_MAP_HIVE_IMPORT`;
+LOAD DATA INPATH 'file:BASEPATH/DECIMAL_MAP_HIVE_IMPORT' INTO TABLE `DECIMAL_MAP_HIVE_IMPORT`;

testdata/hive/scripts/failingImport.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `DATE_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/DATE_HIVE_IMPORT' INTO TABLE `DATE_HIVE_IMPORT`;
+LOAD DATA INPATH 'file:BASEPATH/DATE_HIVE_IMPORT' INTO TABLE `DATE_HIVE_IMPORT`;

testdata/hive/scripts/fieldWithNewlineImport.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `FIELD_WITH_NL_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/FIELD_WITH_NL_HIVE_IMPORT' INTO TABLE `FIELD_WITH_NL_HIVE_IMPORT`;
+LOAD DATA INPATH 'file:BASEPATH/FIELD_WITH_NL_HIVE_IMPORT' INTO TABLE `FIELD_WITH_NL_HIVE_IMPORT`;

testdata/hive/scripts/fieldWithNewlineReplacementImport.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT' INTO TABLE `FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT`;
+LOAD DATA INPATH 'file:BASEPATH/FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT' INTO TABLE `FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT`;

testdata/hive/scripts/incrementalHiveAppend10.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`;
+LOAD DATA INPATH 'file:BASEPATH/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`;

testdata/hive/scripts/incrementalHiveAppend20.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`;
+LOAD DATA INPATH 'file:BASEPATH/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`;

testdata/hive/scripts/incrementalHiveAppendEmpty.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`;
+LOAD DATA INPATH 'file:BASEPATH/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`;

testdata/hive/scripts/normalImport.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `NORMAL_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/NORMAL_HIVE_IMPORT' INTO TABLE `NORMAL_HIVE_IMPORT`;
+LOAD DATA INPATH 'file:BASEPATH/NORMAL_HIVE_IMPORT' INTO TABLE `NORMAL_HIVE_IMPORT`;

testdata/hive/scripts/numericImport.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `NUMERIC_HIVE_IMPORT` ( `DATA_COL0` DOUBLE, `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/NUMERIC_HIVE_IMPORT' INTO TABLE `NUMERIC_HIVE_IMPORT`;
+LOAD DATA INPATH 'file:BASEPATH/NUMERIC_HIVE_IMPORT' INTO TABLE `NUMERIC_HIVE_IMPORT`;

testdata/hive/scripts/partitionImport.q

@@ -14,4 +14,4 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 CREATE TABLE IF NOT EXISTS `PARTITION_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) PARTITIONED BY (ds STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
-LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/PARTITION_HIVE_IMPORT' INTO TABLE `PARTITION_HIVE_IMPORT` PARTITION (ds='20110413');
+LOAD DATA INPATH 'file:BASEPATH/PARTITION_HIVE_IMPORT' INTO TABLE `PARTITION_HIVE_IMPORT` PARTITION (ds='20110413');