SQOOP-3438 Sqoop Import with create hcatalog table for ORC will not work with Hive3 as the table created would be a ACID table and transactional

- Adding negative test case
 - Extending documentation
 - Parameter / option checking

Change-Id: Id2830c3cecc78d77cb83a7f18dc7e2e7b14850e3

src/docs/user/hcatalog.txt

@@ -62,8 +62,11 @@ or export job is done using HCatalog tables, and it is a required option for
 HCatalog jobs.
 
 +--hcatalog-external-table+::
-Set this flag if you need to create external Hive table for example to store
+Use this flag if you need to create external Hive table for example to store
-data in form of ORC files.
+data in non-transactional tables. For e.g. with:
+--hcatalog-storage-stanza "stored as orc tblproperties (\"transactional\"=\"false\")"
+This flag can only be used when
+--create-hcatalog-table or --drop-and-create-hcatalog-table is used.
 
 +--hcatalog-home+::
 The home directory for the HCatalog installation. The directory is

src/java/org/apache/sqoop/SqoopOptions.java

@@ -1656,7 +1656,7 @@ public String getHCatTableName() {
     return this.hCatTableName;
   }
 
-  public void useExternalHCatTable(boolean value) {
+  public void setExternalHCatTable(boolean value) {
     this.isExternalHCatTable = value;
   }
 

src/java/org/apache/sqoop/tool/BaseSqoopTool.java

@@ -1308,7 +1308,7 @@ protected void applyHCatalogOptions(CommandLine in, SqoopOptions out) {
     }
 
     if (in.hasOption(HCATALOG_EXTERNAL_TABLE_ARG)) {
-      out.useExternalHCatTable(true);
+      out.setExternalHCatTable(true);
     }
 
     if (in.hasOption(HCATALOG_DATABASE_ARG)) {
@@ -1597,6 +1597,13 @@ protected void validateHiveOptions(SqoopOptions options)
         + " option." + HELP_STR);
     }
 
+    if(options.isHCatTableExternal() &&
+            !(options.doCreateHCatalogTable() || options.doDropAndCreateHCatalogTable())) {
+      throw new InvalidOptionsException(String.format(
+              "Using --%s only takes effect when --%s or --%s is present",
+              HCATALOG_EXTERNAL_TABLE_ARG, CREATE_HCATALOG_TABLE_ARG, DROP_AND_CREATE_HCATALOG_TABLE));
+    }
+
     if (options.doHiveImport()
         && options.getFileLayout() == SqoopOptions.FileLayout.AvroDataFile) {
       throw new InvalidOptionsException("Hive import is not compatible with "

src/test/org/apache/sqoop/hcat/HCatalogImportTest.java

@@ -50,6 +50,7 @@
 import org.apache.sqoop.hcat.HCatalogTestUtils.CreateMode;
 import org.apache.sqoop.hcat.HCatalogTestUtils.KeyType;
 import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities;
+import org.junit.Assert;
 import org.junit.Before;
 
 import org.apache.sqoop.Sqoop;
@@ -793,6 +794,57 @@ public void testExternalTableCreation() throws Exception {
             null, true, false);
   }
 
+  @Test
+  public void testExternalTableDropAndCreation() throws Exception {
+    final int TOTAL_RECORDS = 1 * 10;
+    String table = getTableName().toUpperCase();
+    ColumnGenerator[] cols = new ColumnGenerator[] {
+            HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
+                    "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
+                    new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
+            HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
+                    "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
+                    new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY),
+    };
+    List<String> addlArgsArray = new ArrayList<String>();
+    addlArgsArray.add("--drop-and-create-hcatalog-table");
+    addlArgsArray.add("--hcatalog-external-table");
+    addlArgsArray.add("--hcatalog-storage-stanza");
+    addlArgsArray.add("\"stored as orc tblproperties (\"transactional\"=\"false\")\"");
+    setExtraArgs(addlArgsArray);
+    utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
+    runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
+            null, true, false);
+  }
+
+  @Test
+  public void testExternalTableCreationFailsIfNoCreateOrDropTablePresent() throws Exception {
+    final int TOTAL_RECORDS = 1 * 10;
+    String table = getTableName().toUpperCase();
+    ColumnGenerator[] cols = new ColumnGenerator[] {
+            HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
+                    "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
+                    new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
+            HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
+                    "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
+                    new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY),
+    };
+    List<String> addlArgsArray = new ArrayList<String>();
+    addlArgsArray.add("--hcatalog-external-table");
+    addlArgsArray.add("--hcatalog-storage-stanza");
+    addlArgsArray.add("\"stored as orc tblproperties (\"transactional\"=\"false\")\"");
+    setExtraArgs(addlArgsArray);
+    utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
+    try {
+      runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
+              null, true, false);
+      Assert.fail("--hcatalog-external-table cannot be used without signing creation of hcatalog table.");
+    } catch (Exception e) {
+      LOG.debug("Caught expected exception while running "
+              + " hcatalog-external-table without signing create or drop-and-create test", e);
+    }
+  }
+
   @Test
   public void testTableCreationWithPartition() throws Exception {
     final int TOTAL_RECORDS = 1 * 10;