SQOOP-3438 Sqoop Import with create hcatalog table for ORC will not work with Hive3 as the table created would be a ACID table and transactional

- implemented reqested test changes
- removed unnecessary refactoring in old code

Change-Id: I8ac7e9b10669615fa479e13e02d27b877bdad462

src/test/org/apache/sqoop/hcat/HCatalogImportTest.java

@@ -62,6 +62,10 @@
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
+import static org.apache.sqoop.tool.BaseSqoopTool.CREATE_HCATALOG_TABLE_ARG;
+import static org.apache.sqoop.tool.BaseSqoopTool.DROP_AND_CREATE_HCATALOG_TABLE;
+import static org.apache.sqoop.tool.BaseSqoopTool.HCATALOG_EXTERNAL_TABLE_ARG;
+import static org.apache.sqoop.tool.BaseSqoopTool.HCATALOG_STORAGE_STANZA_ARG;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -72,10 +76,6 @@
 public class HCatalogImportTest extends ImportJobTestCase {
   private static final Log LOG =
     LogFactory.getLog(HCatalogImportTest.class);
-  public static final String CREATE_HCATALOG_TABLE = "--create-hcatalog-table";
-  public static final String HCATALOG_EXTERNAL_TABLE = "--hcatalog-external-table";
-  public static final String DROP_AND_CREATE_HCATALOG_TABLE ="--drop-and-create-hcatalog-table";
-  public static final String HCATALOG_STORAGE_STANZA = "--hcatalog-storage-stanza";
   public static final String TABLEPROPERTIES_ORC_NONTRANSACTIONAL =
           "\"stored as orc tblproperties (\"transactional\"=\"false\")\"";
   private final HCatalogTestUtils utils = HCatalogTestUtils.instance();
@@ -769,7 +769,7 @@ public void testTableCreation() throws Exception {
         new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY),
     };
     List<String> addlArgsArray = new ArrayList<String>();
-    addlArgsArray.add(CREATE_HCATALOG_TABLE);
+    addlArgsArray.add("--create-hcatalog-table");
     setExtraArgs(addlArgsArray);
     utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
     runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
@@ -779,37 +779,37 @@ public void testTableCreation() throws Exception {
   @Test
   public void testExternalTableCreation() throws Exception {
     externalTableCreationCoreTest(true,
-            CREATE_HCATALOG_TABLE,
+            "--" + CREATE_HCATALOG_TABLE_ARG,
-            HCATALOG_EXTERNAL_TABLE,
+            "--" + HCATALOG_EXTERNAL_TABLE_ARG,
-            HCATALOG_STORAGE_STANZA,
+            "--" + HCATALOG_STORAGE_STANZA_ARG,
             TABLEPROPERTIES_ORC_NONTRANSACTIONAL);
   }
 
   @Test
   public void testExternalTableDropAndCreationWithExistingTargetTable() throws Exception {
-    utils.createHCatTable(CreateMode.CREATE, 10, getTableName().toUpperCase());
+    utils.createHCatExternalTable(getTableName().toUpperCase());
     externalTableCreationCoreTest(false,
-            DROP_AND_CREATE_HCATALOG_TABLE,
+            "--" + DROP_AND_CREATE_HCATALOG_TABLE,
-            HCATALOG_EXTERNAL_TABLE,
+            "--" + HCATALOG_EXTERNAL_TABLE_ARG,
-            HCATALOG_STORAGE_STANZA,
+            "--" + HCATALOG_STORAGE_STANZA_ARG,
             TABLEPROPERTIES_ORC_NONTRANSACTIONAL);
   }
 
-  @Test (expected = Exception.class)
+  @Test (expected = IOException.class)
-  public void testExternalTableCreationWithExistingTargetTable() throws Exception {
+  public void testExternalTableCreationFailsDueToExistingTable() throws Exception {
-    utils.createHCatTable(CreateMode.CREATE, 10, getTableName().toUpperCase());
+    utils.createHCatExternalTable(getTableName().toUpperCase());
     externalTableCreationCoreTest(false,
-            CREATE_HCATALOG_TABLE,
+            "--" + CREATE_HCATALOG_TABLE_ARG,
-            HCATALOG_EXTERNAL_TABLE,
+            "--" + HCATALOG_EXTERNAL_TABLE_ARG,
-            HCATALOG_STORAGE_STANZA,
+            "--" + HCATALOG_STORAGE_STANZA_ARG,
             TABLEPROPERTIES_ORC_NONTRANSACTIONAL);
   }
 
-  @Test(expected = Exception.class)
+  @Test(expected = IOException.class)
   public void testExternalTableCreationFailsIfNoCreateOrDropTablePresent() throws Exception {
     externalTableCreationCoreTest(true,
-            HCATALOG_EXTERNAL_TABLE,
+            "--" + HCATALOG_EXTERNAL_TABLE_ARG,
-            HCATALOG_STORAGE_STANZA,
+            "--" + HCATALOG_STORAGE_STANZA_ARG,
             TABLEPROPERTIES_ORC_NONTRANSACTIONAL);
   }
 
@@ -850,7 +850,7 @@ public void testTableCreationWithPartition() throws Exception {
     addlArgsArray.add("col1");
     addlArgsArray.add("--hive-partition-value");
     addlArgsArray.add("2");
-    addlArgsArray.add(CREATE_HCATALOG_TABLE);
+    addlArgsArray.add("--create-hcatalog-table");
     setExtraArgs(addlArgsArray);
     utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
     runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
@@ -873,7 +873,7 @@ public void testTableCreationWithMultipleStaticPartKeys() throws Exception {
     addlArgsArray.add("col0,col1");
     addlArgsArray.add("--hcatalog-partition-values");
     addlArgsArray.add("1,2");
-    addlArgsArray.add(CREATE_HCATALOG_TABLE);
+    addlArgsArray.add("--create-hcatalog-table");
     setExtraArgs(addlArgsArray);
     utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
     runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
@@ -896,8 +896,8 @@ public void testTableCreationWithStorageStanza() throws Exception {
     addlArgsArray.add("col1");
     addlArgsArray.add("--hive-partition-value");
     addlArgsArray.add("2");
-    addlArgsArray.add(CREATE_HCATALOG_TABLE);
+    addlArgsArray.add("--create-hcatalog-table");
-    addlArgsArray.add(HCATALOG_STORAGE_STANZA);
+    addlArgsArray.add("--hcatalog-storage-stanza");
     addlArgsArray.add(HCatalogTestUtils.STORED_AS_TEXT);
     setExtraArgs(addlArgsArray);
     utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
@@ -987,7 +987,7 @@ public void testCreateTableWithPreExistingTable() throws Exception {
         "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
         new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY), };
     List<String> addlArgsArray = new ArrayList<String>();
-    addlArgsArray.add(CREATE_HCATALOG_TABLE);
+    addlArgsArray.add("--create-hcatalog-table");
     setExtraArgs(addlArgsArray);
     try {
       // Precreate table
@@ -1014,7 +1014,7 @@ public void testDropAndCreateWithPreExistingTable() throws Exception {
         "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
         new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY), };
     List<String> addlArgsArray = new ArrayList<String>();
-    addlArgsArray.add(DROP_AND_CREATE_HCATALOG_TABLE);
+    addlArgsArray.add("--drop-and-create-hcatalog-table");
     setExtraArgs(addlArgsArray);
     // Precreate table
     utils.createHCatTable(CreateMode.CREATE, TOTAL_RECORDS, table, cols);
@@ -1035,7 +1035,7 @@ public void testDropAndCreateWithoutPreExistingTable() throws Exception {
         new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY),
     };
     List<String> addlArgsArray = new ArrayList<String>();
-    addlArgsArray.add(DROP_AND_CREATE_HCATALOG_TABLE);
+    addlArgsArray.add("--drop-and-create-hcatalog-table");
     setExtraArgs(addlArgsArray);
     runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
       null, true, false);
@@ -1065,7 +1065,7 @@ public void testTableCreationWithNonIdentColChars() throws Exception {
         10, KeyType.NOT_A_KEY),
     };
     List<String> addlArgsArray = new ArrayList<String>();
-    addlArgsArray.add(CREATE_HCATALOG_TABLE);
+    addlArgsArray.add("--create-hcatalog-table");
     setExtraArgs(addlArgsArray);
     utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
     runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
@@ -1111,7 +1111,7 @@ public void testPublishTableImportData() throws Exception {
     cfgParams.add(ConfigurationConstants.DATA_PUBLISH_CLASS + "=" + DummyDataPublisher.class.getName());
     setConfigParams(cfgParams);
     List<String> addlArgsArray = new ArrayList<String>();
-    addlArgsArray.add(CREATE_HCATALOG_TABLE);
+    addlArgsArray.add("--create-hcatalog-table");
     setExtraArgs(addlArgsArray);
     utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
     runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);

src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java

@@ -752,6 +752,25 @@ public HCatSchema createHCatTable(CreateMode mode, int count,
     return hCatFullSchema;
   }
 
+  HCatSchema createHCatExternalTable(String table, ColumnGenerator... extraCols)
+          throws Exception {
+    HCatSchema hCatTblSchema = generateHCatTableSchema(extraCols);
+    HCatSchema hCatPartSchema = generateHCatPartitionSchema(extraCols);
+    HCatSchema hCatFullSchema = new HCatSchema(hCatTblSchema.getFields());
+    for (HCatFieldSchema hfs : hCatPartSchema.getFields()) {
+      hCatFullSchema.append(hfs);
+    }
+    String databaseName = SqoopHCatUtilities.DEFHCATDB;
+    String createCmd = getHCatCreateTableCmd(databaseName, table,
+            hCatTblSchema.getFields(), hCatPartSchema.getFields())
+            .replaceFirst(
+                    "create table",
+                    "create external table");
+    utils.launchHCatCli(createCmd);
+    LOG.info("Created HCatalog table " + databaseName + "." + table);
+    return hCatFullSchema;
+  }
+
   private void loadHCatTable(HCatSchema hCatSchema, String table,
     int count, ColumnGenerator... extraCols)
     throws Exception {