From 4e9dd86c184deda10e498ab3acd2400f68b1b7e1 Mon Sep 17 00:00:00 2001 From: Denes Bodo Date: Fri, 10 May 2019 14:39:16 +0200 Subject: [PATCH] SQOOP-3438 Sqoop Import with create hcatalog table for ORC will not work with Hive3 as the table created would be a ACID table and transactional - Adding negative test case - Extending documentation - Parameter / option checking Change-Id: Id2830c3cecc78d77cb83a7f18dc7e2e7b14850e3 --- src/docs/user/hcatalog.txt | 7 ++- src/java/org/apache/sqoop/SqoopOptions.java | 2 +- .../org/apache/sqoop/tool/BaseSqoopTool.java | 9 +++- .../apache/sqoop/hcat/HCatalogImportTest.java | 52 +++++++++++++++++++ 4 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/docs/user/hcatalog.txt b/src/docs/user/hcatalog.txt index e652f8cd..96a90f75 100644 --- a/src/docs/user/hcatalog.txt +++ b/src/docs/user/hcatalog.txt @@ -62,8 +62,11 @@ or export job is done using HCatalog tables, and it is a required option for HCatalog jobs. +--hcatalog-external-table+:: -Set this flag if you need to create external Hive table for example to store -data in form of ORC files. +Use this flag if you need to create external Hive table for example to store +data in non-transactional tables. For e.g. with: +--hcatalog-storage-stanza "stored as orc tblproperties (\"transactional\"=\"false\")" +This flag can only be used when +--create-hcatalog-table or --drop-and-create-hcatalog-table is used. +--hcatalog-home+:: The home directory for the HCatalog installation. The directory is diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index f4c6a4cc..cec81034 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -1656,7 +1656,7 @@ public String getHCatTableName() { return this.hCatTableName; } - public void useExternalHCatTable(boolean value) { + public void setExternalHCatTable(boolean value) { this.isExternalHCatTable = value; } diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 8520a1b2..caf0598d 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -1308,7 +1308,7 @@ protected void applyHCatalogOptions(CommandLine in, SqoopOptions out) { } if (in.hasOption(HCATALOG_EXTERNAL_TABLE_ARG)) { - out.useExternalHCatTable(true); + out.setExternalHCatTable(true); } if (in.hasOption(HCATALOG_DATABASE_ARG)) { @@ -1597,6 +1597,13 @@ protected void validateHiveOptions(SqoopOptions options) + " option." + HELP_STR); } + if(options.isHCatTableExternal() && + !(options.doCreateHCatalogTable() || options.doDropAndCreateHCatalogTable())) { + throw new InvalidOptionsException(String.format( + "Using --%s only takes effect when --%s or --%s is present", + HCATALOG_EXTERNAL_TABLE_ARG, CREATE_HCATALOG_TABLE_ARG, DROP_AND_CREATE_HCATALOG_TABLE)); + } + if (options.doHiveImport() && options.getFileLayout() == SqoopOptions.FileLayout.AvroDataFile) { throw new InvalidOptionsException("Hive import is not compatible with " diff --git a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java index 4a38e2d6..d6eaf7eb 100644 --- a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java +++ b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java @@ -50,6 +50,7 @@ import org.apache.sqoop.hcat.HCatalogTestUtils.CreateMode; import org.apache.sqoop.hcat.HCatalogTestUtils.KeyType; import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities; +import org.junit.Assert; import org.junit.Before; import org.apache.sqoop.Sqoop; @@ -793,6 +794,57 @@ public void testExternalTableCreation() throws Exception { null, true, false); } + @Test + public void testExternalTableDropAndCreation() throws Exception { + final int TOTAL_RECORDS = 1 * 10; + String table = getTableName().toUpperCase(); + ColumnGenerator[] cols = new ColumnGenerator[] { + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0), + "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0, + new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY), + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1), + "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0, + new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY), + }; + List addlArgsArray = new ArrayList(); + addlArgsArray.add("--drop-and-create-hcatalog-table"); + addlArgsArray.add("--hcatalog-external-table"); + addlArgsArray.add("--hcatalog-storage-stanza"); + addlArgsArray.add("\"stored as orc tblproperties (\"transactional\"=\"false\")\""); + setExtraArgs(addlArgsArray); + utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB); + runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, + null, true, false); + } + + @Test + public void testExternalTableCreationFailsIfNoCreateOrDropTablePresent() throws Exception { + final int TOTAL_RECORDS = 1 * 10; + String table = getTableName().toUpperCase(); + ColumnGenerator[] cols = new ColumnGenerator[] { + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0), + "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0, + new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY), + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1), + "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0, + new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY), + }; + List addlArgsArray = new ArrayList(); + addlArgsArray.add("--hcatalog-external-table"); + addlArgsArray.add("--hcatalog-storage-stanza"); + addlArgsArray.add("\"stored as orc tblproperties (\"transactional\"=\"false\")\""); + setExtraArgs(addlArgsArray); + utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB); + try { + runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, + null, true, false); + Assert.fail("--hcatalog-external-table cannot be used without signing creation of hcatalog table."); + } catch (Exception e) { + LOG.debug("Caught expected exception while running " + + " hcatalog-external-table without signing create or drop-and-create test", e); + } + } + @Test public void testTableCreationWithPartition() throws Exception { final int TOTAL_RECORDS = 1 * 10;