github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 03:59:18 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3438 Sqoop Import with create hcatalog table for ORC will not work with Hive3 as the table created would be a ACID table and transactional

Browse Source 
- Adding negative test case
 - Extending documentation
 - Parameter / option checking

Change-Id: Id2830c3cecc78d77cb83a7f18dc7e2e7b14850e3
This commit is contained in:
Denes Bodo 2019-05-10 14:39:16 +02:00
parent ec707d0b38
commit 4e9dd86c18
4 changed files with 66 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/docs/user/hcatalog.txt
View File

@ -62,8 +62,11 @@ or export job is done using HCatalog tables, and it is a required option for
HCatalog jobs.
+--hcatalog-external-table+::
Set this flag if you need to create external Hive table for example to store
data in form of ORC files.
Use this flag if you need to create external Hive table for example to store
data in non-transactional tables. For e.g. with:
--hcatalog-storage-stanza "stored as orc tblproperties (\"transactional\"=\"false\")"
This flag can only be used when
--create-hcatalog-table or --drop-and-create-hcatalog-table is used.
+--hcatalog-home+::
The home directory for the HCatalog installation. The directory is

2
src/java/org/apache/sqoop/SqoopOptions.java
View File

@ -1656,7 +1656,7 @@ public String getHCatTableName() {
return this.hCatTableName;
}
public void useExternalHCatTable(boolean value) {
public void setExternalHCatTable(boolean value) {
this.isExternalHCatTable = value;
}

9
src/java/org/apache/sqoop/tool/BaseSqoopTool.java
View File

@ -1308,7 +1308,7 @@ protected void applyHCatalogOptions(CommandLine in, SqoopOptions out) {
}
if (in.hasOption(HCATALOG_EXTERNAL_TABLE_ARG)) {
out.useExternalHCatTable(true);
out.setExternalHCatTable(true);
}
if (in.hasOption(HCATALOG_DATABASE_ARG)) {
@ -1597,6 +1597,13 @@ protected void validateHiveOptions(SqoopOptions options)
+ " option." + HELP_STR);
}
if(options.isHCatTableExternal() &&
!(options.doCreateHCatalogTable() || options.doDropAndCreateHCatalogTable())) {
throw new InvalidOptionsException(String.format(
"Using --%s only takes effect when --%s or --%s is present",
HCATALOG_EXTERNAL_TABLE_ARG, CREATE_HCATALOG_TABLE_ARG, DROP_AND_CREATE_HCATALOG_TABLE));
}
if (options.doHiveImport()
&& options.getFileLayout() == SqoopOptions.FileLayout.AvroDataFile) {
throw new InvalidOptionsException("Hive import is not compatible with "

52
src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
View File

@ -50,6 +50,7 @@
import org.apache.sqoop.hcat.HCatalogTestUtils.CreateMode;
import org.apache.sqoop.hcat.HCatalogTestUtils.KeyType;
import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities;
import org.junit.Assert;
import org.junit.Before;
import org.apache.sqoop.Sqoop;
@ -793,6 +794,57 @@ public void testExternalTableCreation() throws Exception {
null, true, false);
}
@Test
public void testExternalTableDropAndCreation() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
ColumnGenerator[] cols = new ColumnGenerator[] {
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
"varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
"varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY),
};
List<String> addlArgsArray = new ArrayList<String>();
addlArgsArray.add("--drop-and-create-hcatalog-table");
addlArgsArray.add("--hcatalog-external-table");
addlArgsArray.add("--hcatalog-storage-stanza");
addlArgsArray.add("\"stored as orc tblproperties (\"transactional\"=\"false\")\"");
setExtraArgs(addlArgsArray);
utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
null, true, false);
}
@Test
public void testExternalTableCreationFailsIfNoCreateOrDropTablePresent() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
ColumnGenerator[] cols = new ColumnGenerator[] {
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
"varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
"varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY),
};
List<String> addlArgsArray = new ArrayList<String>();
addlArgsArray.add("--hcatalog-external-table");
addlArgsArray.add("--hcatalog-storage-stanza");
addlArgsArray.add("\"stored as orc tblproperties (\"transactional\"=\"false\")\"");
setExtraArgs(addlArgsArray);
utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
try {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
null, true, false);
Assert.fail("--hcatalog-external-table cannot be used without signing creation of hcatalog table.");
} catch (Exception e) {
LOG.debug("Caught expected exception while running "
+ " hcatalog-external-table without signing create or drop-and-create test", e);
}
}
@Test
public void testTableCreationWithPartition() throws Exception {
final int TOTAL_RECORDS = 1 * 10;