diff --git a/src/docs/user/hcatalog.txt b/src/docs/user/hcatalog.txt index 2ae1d54d..e652f8cd 100644 --- a/src/docs/user/hcatalog.txt +++ b/src/docs/user/hcatalog.txt @@ -61,6 +61,10 @@ The presence of the +--hcatalog-table+ option signifies that the import or export job is done using HCatalog tables, and it is a required option for HCatalog jobs. ++--hcatalog-external-table+:: +Set this flag if you need to create external Hive table for example to store +data in form of ORC files. + +--hcatalog-home+:: The home directory for the HCatalog installation. The directory is expected to have a +lib+ subdirectory and a +share/hcatalog+ subdirectory diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index a7c19daa..f4c6a4cc 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -240,6 +240,8 @@ public String toString() { @StoredAsProperty("hive.partition.value") private String hivePartitionValue; @StoredAsProperty("hcatalog.table.name") private String hCatTableName; + @StoredAsProperty("hcatalog.external.table") + private boolean isExternalHCatTable = false; @StoredAsProperty("hcatalog.database.name") private String hCatDatabaseName; @StoredAsProperty("hcatalog.create.table") @@ -1654,6 +1656,14 @@ public String getHCatTableName() { return this.hCatTableName; } + public void useExternalHCatTable(boolean value) { + this.isExternalHCatTable = value; + } + + public boolean isHCatTableExternal() { + return this.isExternalHCatTable; + } + public void setHCatDatabaseName(String hd) { this.hCatDatabaseName = hd; } diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java index 234b7a87..cd939337 100644 --- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java +++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java @@ -377,11 +377,11 @@ public void configureHCat(final SqoopOptions opts, final Job job, if (options.doCreateHCatalogTable()) { LOG.info("Creating HCatalog table " + hCatQualifiedTableName + " for import"); - createHCatTable(false); + createHCatTable(false, options.isHCatTableExternal()); } else if (options.doDropAndCreateHCatalogTable()) { LOG.info("Dropping and Creating HCatalog table " + hCatQualifiedTableName + " for import"); - createHCatTable(true); + createHCatTable(true, options.isHCatTableExternal()); } // For serializing the schema to conf HCatInputFormat hif = HCatInputFormat.setInput(hCatJob, hCatDatabaseName, @@ -599,14 +599,18 @@ public static StringBuilder escHCatObj(String objectName) { return sb; } - private void createHCatTable(boolean dropIfExists) throws IOException { + private void createHCatTable(boolean dropIfExists, boolean isExternal) throws IOException { StringBuilder sb = new StringBuilder(); if (dropIfExists) { sb.append("drop table "). append(escHCatObj(hCatDatabaseName)).append('.'). append(escHCatObj(hCatTableName)).append(";\n"); } - sb.append("create table "). + sb.append("create "); + if(isExternal) { + sb.append("external "); + } + sb.append("table "). append(escHCatObj(hCatDatabaseName)).append('.'); sb.append(escHCatObj(hCatTableName)).append(" (\n\t"); boolean first = true; diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 96f06de0..8520a1b2 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -132,6 +132,7 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool { public static final String CREATE_HIVE_TABLE_ARG = "create-hive-table"; public static final String HCATALOG_TABLE_ARG = "hcatalog-table"; + public static final String HCATALOG_EXTERNAL_TABLE_ARG = "hcatalog-external-table"; public static final String HCATALOG_DATABASE_ARG = "hcatalog-database"; public static final String CREATE_HCATALOG_TABLE_ARG = "create-hcatalog-table"; @@ -661,6 +662,10 @@ protected RelatedOptions getHCatalogOptions() { .withDescription("HCatalog table name") .withLongOpt(HCATALOG_TABLE_ARG) .create()); + hCatOptions.addOption(OptionBuilder + .withDescription("Signing that HCatalog table shall be created as external") + .withLongOpt(HCATALOG_EXTERNAL_TABLE_ARG) + .create()); hCatOptions.addOption(OptionBuilder .hasArg() .withDescription("HCatalog database name") @@ -1302,6 +1307,10 @@ protected void applyHCatalogOptions(CommandLine in, SqoopOptions out) { out.setHCatTableName(in.getOptionValue(HCATALOG_TABLE_ARG)); } + if (in.hasOption(HCATALOG_EXTERNAL_TABLE_ARG)) { + out.useExternalHCatTable(true); + } + if (in.hasOption(HCATALOG_DATABASE_ARG)) { out.setHCatDatabaseName(in.getOptionValue(HCATALOG_DATABASE_ARG)); }