From eac50d0e2457699ada83b56fbce35deb2d4e3b97 Mon Sep 17 00:00:00 2001 From: Andrew Bayer Date: Fri, 22 Jul 2011 20:04:38 +0000 Subject: [PATCH] SQOOP-212. Hive import for existing table does not work. This patch fixes a bug that prevents importing data into an existing hive table with the 'hive-overwrite' argument set. From: Ahmed Radwan git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1150043 13f79535-47bb-0310-9956-ffa450edef68 --- src/docs/man/hive-args.txt | 7 +++++-- src/docs/man/sqoop-create-hive-table.txt | 7 +++++-- src/docs/user/create-hive-table.txt | 4 +++- src/docs/user/hive-args.txt | 2 ++ src/java/com/cloudera/sqoop/SqoopOptions.java | 14 ++++++++++++++ .../com/cloudera/sqoop/hive/TableDefWriter.java | 9 ++++++--- .../com/cloudera/sqoop/tool/BaseSqoopTool.java | 10 ++++++++++ .../com/cloudera/sqoop/hive/TestHiveImport.java | 2 +- 8 files changed, 46 insertions(+), 9 deletions(-) diff --git a/src/docs/man/hive-args.txt b/src/docs/man/hive-args.txt index 6c8d6230..b8f3ea34 100644 --- a/src/docs/man/hive-args.txt +++ b/src/docs/man/hive-args.txt @@ -26,8 +26,11 @@ Hive options If set, then import the table into Hive --hive-overwrite:: - Overwrites existing table in hive. - By default it does not overwrite existing table. + Overwrites existing data in the hive table if it exists. + +--create-hive-table:: + If set, then the job will fail if the target hive table exits. + By default this property is false. --hive-table (table-name):: When used with --hive-import, overrides the destination table name diff --git a/src/docs/man/sqoop-create-hive-table.txt b/src/docs/man/sqoop-create-hive-table.txt index 39fbea92..93e43016 100644 --- a/src/docs/man/sqoop-create-hive-table.txt +++ b/src/docs/man/sqoop-create-hive-table.txt @@ -32,8 +32,11 @@ Hive options Override $HIVE_HOME --hive-overwrite:: - Overwrites existing table in hive. - By default it does not overwrite existing table. + Overwrites existing data in the hive table if it exists. + +--create-hive-table:: + If set, then the job will fail if the target hive table exits. + By default this property is false. --hive-table (table-name):: When used with --hive-import, overrides the destination table name diff --git a/src/docs/user/create-hive-table.txt b/src/docs/user/create-hive-table.txt index 9ff32899..7c3c2d5e 100644 --- a/src/docs/user/create-hive-table.txt +++ b/src/docs/user/create-hive-table.txt @@ -48,10 +48,12 @@ Argument Description -------------------------------------------------------------------------- +\--hive-home + Override +$HIVE_HOME+ +\--hive-overwrite+ Overwrite existing data in the Hive table. ++\--create-hive-table+ If set, then the job will fail if the target hive + table exits. By default this property is false. +\--hive-table + Sets the table name to use when importing \ to Hive. +\--table+ The database table to read the \ - definition from. + definition from. -------------------------------------------------------------------------- include::output-args.txt[] diff --git a/src/docs/user/hive-args.txt b/src/docs/user/hive-args.txt index c5af987d..7e6b7a01 100644 --- a/src/docs/user/hive-args.txt +++ b/src/docs/user/hive-args.txt @@ -26,6 +26,8 @@ Argument Description +\--hive-import+ Import tables into Hive (Uses Hive's \ default delimiters if none are set.) +\--hive-overwrite+ Overwrite existing data in the Hive table. ++\--create-hive-table+ If set, then the job will fail if the target hive + table exits. By default this property is false. +\--hive-table + Sets the table name to use when importing\ to Hive. +\--hive-drop-import-delims+ Drops '\n', '\r', and '\01' from string\ diff --git a/src/java/com/cloudera/sqoop/SqoopOptions.java b/src/java/com/cloudera/sqoop/SqoopOptions.java index e3abe5ae..eddae34e 100644 --- a/src/java/com/cloudera/sqoop/SqoopOptions.java +++ b/src/java/com/cloudera/sqoop/SqoopOptions.java @@ -145,6 +145,8 @@ public enum IncrementalMode { private String hiveHome; // not serialized to metastore. @StoredAsProperty("hive.import") private boolean hiveImport; @StoredAsProperty("hive.overwrite.table") private boolean overwriteHiveTable; + @StoredAsProperty("hive.fail.table.exists") + private boolean failIfHiveTableExists; @StoredAsProperty("hive.table.name") private String hiveTableName; @StoredAsProperty("hive.drop.delims") private boolean hiveDropDelims; @StoredAsProperty("hive.partition.key") private String hivePartitionKey; @@ -1010,6 +1012,18 @@ public void setHiveDropDelims(boolean dropHiveDelims) { this.hiveDropDelims = dropHiveDelims; } + /** + * @return the user-specified option to specify sqoop's behavior during + * target table creation if the table exists. + */ + public boolean doFailIfHiveTableExists() { + return failIfHiveTableExists; + } + + public void setFailIfHiveTableExists(boolean fail) { + this.failIfHiveTableExists = fail; + } + /** * @return location where .java files go; guaranteed to end with '/'. */ diff --git a/src/java/com/cloudera/sqoop/hive/TableDefWriter.java b/src/java/com/cloudera/sqoop/hive/TableDefWriter.java index 5f889fbd..7dd91352 100644 --- a/src/java/com/cloudera/sqoop/hive/TableDefWriter.java +++ b/src/java/com/cloudera/sqoop/hive/TableDefWriter.java @@ -129,7 +129,7 @@ public String getCreateTableStmt() throws IOException { String [] colNames = getColumnNames(); StringBuilder sb = new StringBuilder(); - if (options.doOverwriteHiveTable()) { + if (options.doFailIfHiveTableExists()) { sb.append("CREATE TABLE `").append(outputTableName).append("` ( "); } else { sb.append("CREATE TABLE IF NOT EXISTS `"); @@ -209,8 +209,11 @@ public String getLoadDataStmt() throws IOException { StringBuilder sb = new StringBuilder(); sb.append("LOAD DATA INPATH '"); - sb.append(finalPathStr); - sb.append("' INTO TABLE `"); + sb.append(finalPathStr + "'"); + if (options.doOverwriteHiveTable()) { + sb.append(" OVERWRITE"); + } + sb.append(" INTO TABLE `"); sb.append(outputTableName); sb.append('`'); diff --git a/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java b/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java index 6cafaa08..2bc0fe9a 100644 --- a/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java +++ b/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java @@ -92,6 +92,8 @@ public abstract class BaseSqoopTool extends SqoopTool { public static final String HIVE_DROP_DELIMS_ARG = "hive-drop-import-delims"; public static final String HIVE_PARTITION_KEY_ARG = "hive-partition-key"; public static final String HIVE_PARTITION_VALUE_ARG = "hive-partition-value"; + public static final String CREATE_HIVE_TABLE_ARG = + "create-hive-table"; public static final String NUM_MAPPERS_ARG = "num-mappers"; public static final String NUM_MAPPERS_SHORT_ARG = "m"; public static final String COMPRESS_ARG = "compress"; @@ -398,6 +400,10 @@ protected RelatedOptions getHiveOptions(boolean explicitHiveImport) { .withDescription("Overwrite existing data in the Hive table") .withLongOpt(HIVE_OVERWRITE_ARG) .create()); + hiveOpts.addOption(OptionBuilder + .withDescription("Fail if the target hive table exists") + .withLongOpt(CREATE_HIVE_TABLE_ARG) + .create()); hiveOpts.addOption(OptionBuilder.withArgName("table-name") .hasArg() .withDescription("Sets the table name to use when importing to hive") @@ -669,6 +675,10 @@ protected void applyHiveOptions(CommandLine in, SqoopOptions out) out.setOverwriteHiveTable(true); } + if (in.hasOption(CREATE_HIVE_TABLE_ARG)) { + out.setFailIfHiveTableExists(true); + } + if (in.hasOption(HIVE_TABLE_ARG)) { out.setHiveTableName(in.getOptionValue(HIVE_TABLE_ARG)); } diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java index 375d4d77..ce48aabe 100644 --- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java +++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java @@ -245,7 +245,7 @@ public void testCreateOverwriteHiveImport() throws IOException { setNumCols(3); String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" }; String [] vals = { "'test'", "42", "'somestring'" }; - String [] extraArgs = {"--hive-overwrite"}; + String [] extraArgs = {"--hive-overwrite", "--create-hive-table"}; runImportTest(TABLE_NAME, types, vals, "createOverwriteImport.q", getCreateHiveTableArgs(extraArgs), new CreateHiveTableTool());