diff --git a/build.xml b/build.xml
index b1545b66..6e8d3939 100644
--- a/build.xml
+++ b/build.xml
@@ -194,11 +194,12 @@
-
+
+
diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
index ebe72092..2101b068 100644
--- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
+++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
@@ -40,6 +40,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.HadoopShims.HCatHadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -56,6 +59,7 @@
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.hcatalog.common.HCatConstants;
+import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.data.DefaultHCatRecord;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
@@ -66,7 +70,6 @@
import org.apache.sqoop.config.ConfigurationHelper;
import org.apache.sqoop.hive.HiveTypes;
import org.apache.sqoop.manager.ConnManager;
-import org.apache.sqoop.tool.BaseSqoopTool;
import org.apache.sqoop.util.Executor;
import org.apache.sqoop.util.LoggingAsyncSink;
import org.apache.sqoop.util.SubprocessSecurityManager;
@@ -158,6 +161,43 @@ public final class SqoopHCatUtilities {
private static boolean testMode = false;
+ public static boolean isHCatView(final SqoopOptions opts) {
+
+ String hCatDatabaseName = getHCatDbNameFromOptions(opts);
+
+ String hCatTableName = getHCatTableNameFromOptions(opts);
+
+ Configuration conf = opts.getConf();
+ HiveConf hiveConf;
+ try {
+ if (conf != null) {
+ hiveConf = HCatUtil.getHiveConf(conf);
+ } else {
+ hiveConf = new HiveConf(HCatInputFormat.class);
+ }
+ HiveMetaStoreClient client = HCatUtil.getHiveClient(hiveConf);
+ Table table = HCatUtil.getTable(client, hCatDatabaseName, hCatTableName);
+ return table.isView();
+ }
+ catch(Exception e){
+ LOG.warn("We were not able to determine if "+hCatDatabaseName+ ":"+hCatTableName+ "is view or table.");
+ LOG.info("isHCatView threw an exception:", e);
+ return false;
+ }
+ }
+
+ private static String getHCatTableNameFromOptions(SqoopOptions opts) {
+ String optHCTabName = opts.getHCatTableName();
+ return optHCTabName.toLowerCase();
+ }
+
+ private static String getHCatDbNameFromOptions(SqoopOptions opts) {
+ String hCatDatabaseName = opts.getHCatDatabaseName() != null ? opts
+ .getHCatDatabaseName() : DEFHCATDB;
+ hCatDatabaseName = hCatDatabaseName.toLowerCase();
+ return hCatDatabaseName;
+ }
+
static class IntArrayWritable extends ArrayWritable {
public IntArrayWritable() {
super(IntWritable.class);
@@ -278,12 +318,10 @@ public void configureHCat(final SqoopOptions opts, final Job job,
dbTableName = dbTable;
configuration = config;
hCatJob = job;
- hCatDatabaseName = options.getHCatDatabaseName() != null ? options
- .getHCatDatabaseName() : DEFHCATDB;
- hCatDatabaseName = hCatDatabaseName.toLowerCase();
+ hCatDatabaseName = getHCatDbNameFromOptions(opts);
String optHCTabName = options.getHCatTableName();
- hCatTableName = optHCTabName.toLowerCase();
+ hCatTableName = getHCatTableNameFromOptions(opts);
if (!hCatTableName.equals(optHCTabName)) {
LOG.warn("Provided HCatalog table name " + optHCTabName
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 468bf346..235f67a1 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -33,6 +33,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
+import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities;
import org.apache.sqoop.util.CredentialsUtil;
import org.apache.sqoop.util.LoggingUtils;
import org.apache.sqoop.util.password.CredentialProviderHelper;
@@ -1560,6 +1561,9 @@ protected void validateHCatalogOptions(SqoopOptions options)
}
return;
}
+ if(isSet(options.getHCatTableName()) && SqoopHCatUtilities.isHCatView(options)){
+ throw new InvalidOptionsException("Reads/Writes from and to Views are not supported by HCatalog");
+ }
if (options.explicitInputDelims()) {
LOG.warn("Input field/record delimiter options are not "
@@ -1672,6 +1676,10 @@ protected void validateHCatalogOptions(SqoopOptions options)
}
}
+ private boolean isSet(String option) {
+ return org.apache.commons.lang.StringUtils.isNotBlank(option);
+ }
+
protected void validateHBaseOptions(SqoopOptions options)
throws InvalidOptionsException {
if ((options.getHBaseColFamily() != null && options.getHBaseTable() == null)
diff --git a/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java
index ff3dde1c..5ef9b2b8 100644
--- a/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java
+++ b/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java
@@ -32,8 +32,11 @@
import java.util.Map;
import java.util.TimeZone;
+import com.cloudera.sqoop.SqoopOptions;
+import junit.framework.JUnit4TestAdapter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
@@ -46,14 +49,24 @@
import org.junit.Before;
import com.cloudera.sqoop.testutil.ExportJobTestCase;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
/**
* Test that we can export HCatalog tables into databases.
*/
+@RunWith(JUnit4.class)
public class HCatalogExportTest extends ExportJobTestCase {
private static final Log LOG =
LogFactory.getLog(HCatalogExportTest.class);
private HCatalogTestUtils utils = HCatalogTestUtils.instance();
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
@Before
@Override
public void setUp() {
@@ -142,6 +155,7 @@ protected void runHCatExport(List addlArgsArray,
}
}
+ @Test
public void testIntTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -166,6 +180,7 @@ public void testIntTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testExportWithColumnNameValue() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -182,7 +197,7 @@ public void testExportWithColumnNameValue() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
-
+ @Test
public void testFloatTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -201,6 +216,7 @@ public void testFloatTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testNumberTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -220,6 +236,7 @@ public void testNumberTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testDateTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -248,6 +265,7 @@ public void testDateTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testDateTypesToBigInt() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
long offset = TimeZone.getDefault().getRawOffset();
@@ -270,6 +288,7 @@ public void testDateTypesToBigInt() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testStringTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -293,7 +312,7 @@ public void testStringTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
-
+ @Test
public void testBinaryTypes() throws Exception {
ByteBuffer bb = ByteBuffer.wrap(new byte[] { 0, 1, 2 });
final int TOTAL_RECORDS = 1 * 10;
@@ -310,6 +329,7 @@ public void testBinaryTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testColumnProjection() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -324,6 +344,8 @@ public void testColumnProjection() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+
+ @Test
public void testStaticPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -341,6 +363,7 @@ public void testStaticPartitioning() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testStaticPartitioningWithMultipleKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -361,6 +384,7 @@ public void testStaticPartitioningWithMultipleKeys() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -374,6 +398,7 @@ public void testDynamicPartitioning() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testStaticAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -394,6 +419,7 @@ public void testStaticAndDynamicPartitioning() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testMultipleStaticKeysAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -420,6 +446,7 @@ public void testMultipleStaticKeysAndDynamicPartitioning() throws Exception {
/**
* Test other file formats.
*/
+ @Test
public void testSequenceFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -440,6 +467,7 @@ public void testSequenceFile() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testTextFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -460,6 +488,7 @@ public void testTextFile() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
+ @Test
public void testPublishExportJobData() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -479,4 +508,63 @@ public void testPublishExportJobData() throws Exception {
assert (DummyDataPublisher.storeType.equals("hsqldb"));
assert (DummyDataPublisher.operation.equals("export"));
}
+
+ @Test
+ public void testWeCanTellIfHCatViewOrTable() throws Exception {
+ String tableName = getTableName().toUpperCase();
+ String viewName = "view";
+ SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
+ createHCatTableAndView(tableName, viewName, utils);
+
+ Configuration conf = getConf();
+ conf.set("oraoop.disabled", "true");
+ SqoopOptions opts = getSqoopOptions(conf);
+ opts.setHCatTableName(tableName);
+
+ assertFalse(utils.isHCatView(opts));
+ opts.setHCatTableName(viewName);
+ assertTrue(utils.isHCatView(opts));
+ }
+
+ @Test
+ public void testExportHCatViewThrowsException() throws Exception {
+ String tableName = getTableName().toUpperCase();
+ String viewName = "view";
+ SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
+ createHCatTableAndView(tableName, viewName, utils);
+
+ ArrayList args = new ArrayList();
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--hcatalog-table");
+ args.add(viewName);
+ args.add("--connect");
+ args.add(getConnectString());
+
+ args.toArray(new String[0]);
+
+ SqoopHCatUtilities.instance().setConfigured(false);
+ exception.expect(java.io.IOException.class);
+ runExport(args.toArray(new String[0]));
+ }
+
+ private void createHCatTableAndView(String tableName, String viewName, SqoopHCatUtilities utils) throws Exception {
+ HCatalogTestUtils testUtils = HCatalogTestUtils.instance();
+ ColumnGenerator[] cols = new ColumnGenerator[]{
+ HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
+ "boolean", Types.BOOLEAN, HCatFieldSchema.Type.BOOLEAN, 0, 0,
+ Boolean.TRUE, Boolean.TRUE, KeyType.NOT_A_KEY),
+ HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
+ "int", Types.INTEGER, HCatFieldSchema.Type.INT, 5, 5, 10,
+ 10, KeyType.NOT_A_KEY)
+ };
+ testUtils.createHCatTable(CreateMode.CREATE_AND_LOAD, 10, tableName, cols);
+ String createViewCmd = "drop view " + viewName + "; create view " + viewName + " as select * from " + tableName;
+ utils.launchHCatCli(createViewCmd);
+ }
+
+ public static junit.framework.Test suite() {
+ return new JUnit4TestAdapter(HCatalogImportTest.class);
+ }
+
}
diff --git a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
index fe5295a4..28f83f27 100644
--- a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
+++ b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
@@ -35,6 +35,7 @@
import java.util.Map;
import java.util.TimeZone;
+import junit.framework.JUnit4TestAdapter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -57,10 +58,16 @@
import com.cloudera.sqoop.testutil.ImportJobTestCase;
import com.cloudera.sqoop.tool.ImportTool;
import com.cloudera.sqoop.tool.SqoopTool;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
/**
* Test that we can export HCatalog tables into databases.
*/
+@RunWith(JUnit4.class)
public class HCatalogImportTest extends ImportJobTestCase {
private static final Log LOG =
LogFactory.getLog(HCatalogImportTest.class);
@@ -68,6 +75,9 @@ public class HCatalogImportTest extends ImportJobTestCase {
private List extraTestArgs = null;
private List configParams = null;
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
@Override
@Before
public void setUp() {
@@ -343,6 +353,7 @@ protected void runHCatImport(List addlArgsArray,
validateHCatRecords(recs, tblSchema, 10, cols);
}
+ @Test
public void testIntTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -368,6 +379,7 @@ public void testIntTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testFloatTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -387,6 +399,7 @@ public void testFloatTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testNumberTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -407,6 +420,7 @@ public void testNumberTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testDateTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -436,6 +450,7 @@ public void testDateTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testDateTypesToBigInt() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
long offset = TimeZone.getDefault().getRawOffset();
@@ -459,6 +474,7 @@ public void testDateTypesToBigInt() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testStringTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -483,14 +499,16 @@ public void testStringTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testBinaryTypes() throws Exception {
ByteBuffer bb = ByteBuffer.wrap(new byte[] { 0, 1, 2 });
+ ByteBuffer bb10 = ByteBuffer.wrap(new byte[]{0, 1, 2, 0, 0, 0, 0, 0, 0, 0});
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
ColumnGenerator[] cols = new ColumnGenerator[] {
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
"binary(10)", Types.BINARY, HCatFieldSchema.Type.BINARY, 0, 0,
- bb.array(), bb.array(), KeyType.NOT_A_KEY),
+ bb10.array(), bb10.array(), KeyType.NOT_A_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
"longvarbinary", Types.BINARY, HCatFieldSchema.Type.BINARY, 0, 0,
bb.array(), bb.array(), KeyType.NOT_A_KEY),
@@ -500,6 +518,7 @@ public void testBinaryTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testColumnProjection() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -518,6 +537,7 @@ public void testColumnProjection() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, colNames);
}
+ @Test
public void testColumnProjectionMissingPartKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -543,6 +563,7 @@ public void testColumnProjectionMissingPartKeys() throws Exception {
LOG.info("Exception stack trace = " + sw);
}
}
+ @Test
public void testStaticPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -560,6 +581,7 @@ public void testStaticPartitioning() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testStaticPartitioningWithMultipleKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -580,6 +602,7 @@ public void testStaticPartitioningWithMultipleKeys() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -593,6 +616,7 @@ public void testDynamicPartitioning() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testStaticAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -613,6 +637,7 @@ public void testStaticAndDynamicPartitioning() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testMultipleStaticKeysAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -639,6 +664,7 @@ public void testMultipleStaticKeysAndDynamicPartitioning() throws Exception {
/**
* Test other file formats.
*/
+ @Test
public void testSequenceFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -659,6 +685,7 @@ public void testSequenceFile() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testTextFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -680,6 +707,7 @@ public void testTextFile() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testTableCreation() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -694,10 +722,12 @@ public void testTableCreation() throws Exception {
List addlArgsArray = new ArrayList();
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
null, true, false);
}
+ @Test
public void testTableCreationWithPartition() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -716,9 +746,11 @@ public void testTableCreationWithPartition() throws Exception {
addlArgsArray.add("2");
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
+ @Test
public void testTableCreationWithMultipleStaticPartKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -737,9 +769,11 @@ public void testTableCreationWithMultipleStaticPartKeys() throws Exception {
addlArgsArray.add("1,2");
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
+ @Test
public void testTableCreationWithStorageStanza() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -760,9 +794,11 @@ public void testTableCreationWithStorageStanza() throws Exception {
addlArgsArray.add("--hcatalog-storage-stanza");
addlArgsArray.add(HCatalogTestUtils.STORED_AS_TEXT);
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
+ @Test
public void testHiveDropDelims() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -780,6 +816,7 @@ public void testHiveDropDelims() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testHiveDelimsReplacement() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -798,6 +835,7 @@ public void testHiveDelimsReplacement() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testDynamicKeyInMiddle() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -814,6 +852,7 @@ public void testDynamicKeyInMiddle() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testQueryImport() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -830,6 +869,7 @@ public void testQueryImport() throws Exception {
runHCatQueryImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+ @Test
public void testCreateTableWithPreExistingTable() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -856,6 +896,7 @@ public void testCreateTableWithPreExistingTable() throws Exception {
}
}
+ @Test
public void testDropAndCreateWithPreExistingTable() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -875,6 +916,7 @@ public void testDropAndCreateWithPreExistingTable() throws Exception {
null, true, false);
}
+ @Test
public void testDropAndCreateWithoutPreExistingTable() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -893,6 +935,7 @@ public void testDropAndCreateWithoutPreExistingTable() throws Exception {
null, true, false);
}
+ @Test
public void testTableWithNonIdentColChars() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -905,6 +948,8 @@ public void testTableWithNonIdentColChars() throws Exception {
setExtraArgs(addlArgsArray);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
+
+ @Test
public void testTableCreationWithNonIdentColChars() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -916,10 +961,12 @@ public void testTableCreationWithNonIdentColChars() throws Exception {
List addlArgsArray = new ArrayList();
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
null, true, false);
}
+ @Test
public void testPublishQueryImportData() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -941,6 +988,7 @@ public void testPublishQueryImportData() throws Exception {
assert (DummyDataPublisher.storeTable.equals(getTableName()));
}
+ @Test
public void testPublishTableImportData() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@@ -959,9 +1007,70 @@ public void testPublishTableImportData() throws Exception {
List addlArgsArray = new ArrayList();
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
assert (DummyDataPublisher.storeType.equals("hsqldb"));
assert (DummyDataPublisher.operation.equals("import"));
assert (DummyDataPublisher.storeTable.equals(getTableName()));
}
+
+ @Test
+ public void testWeCanTellIfHCatViewOrTable() throws Exception {
+ String tableName = getTableName().toUpperCase();
+ String viewName = "view";
+ SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
+ createHCatTableAndView(tableName, viewName, utils);
+
+ Configuration conf = getConf();
+ conf.set("oraoop.disabled", "true");
+ SqoopOptions opts = getSqoopOptions(conf);
+ opts.setHCatTableName(tableName);
+
+ assertFalse(utils.isHCatView(opts));
+ opts.setHCatTableName(viewName);
+ assertTrue(utils.isHCatView(opts));
+ }
+
+ @Test
+ public void testImportHCatViewThrowsException() throws Exception {
+ String tableName = getTableName().toUpperCase();
+ String viewName = "view";
+ SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
+ createHCatTableAndView(tableName, viewName, utils);
+
+ ArrayList args = new ArrayList();
+
+ args.addAll(getConfigParams());
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--hcatalog-table");
+ args.add(viewName);
+ args.add("--connect");
+ args.add(getConnectString());
+
+ args.toArray(new String[0]);
+
+ SqoopHCatUtilities.instance().setConfigured(false);
+ exception.expect(java.io.IOException.class);
+ runImport(new ImportTool(), args.toArray(new String[0]));
+ }
+
+ private void createHCatTableAndView(String tableName, String viewName, SqoopHCatUtilities utils) throws Exception {
+ HCatalogTestUtils testUtils = HCatalogTestUtils.instance();
+ ColumnGenerator[] cols = new ColumnGenerator[]{
+ HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
+ "boolean", Types.BOOLEAN, HCatFieldSchema.Type.BOOLEAN, 0, 0,
+ Boolean.TRUE, Boolean.TRUE, KeyType.NOT_A_KEY),
+ HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
+ "int", Types.INTEGER, HCatFieldSchema.Type.INT, 5, 5, 10,
+ 10, KeyType.NOT_A_KEY)
+ };
+ testUtils.createHCatTable(CreateMode.CREATE_AND_LOAD, 10, tableName, cols);
+ String createViewCmd = "drop view " + viewName + "; create view " + viewName + " as select * from " + tableName;
+ utils.launchHCatCli(createViewCmd);
+ }
+
+ public static junit.framework.Test suite() {
+ return new JUnit4TestAdapter(HCatalogImportTest.class);
+ }
}
diff --git a/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java b/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
index fb4a200f..4301a334 100644
--- a/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
+++ b/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
@@ -177,29 +177,33 @@ private static String getHCatCreateTableCmd(String dbName,
* memory list.
*/
public void createHCatTableUsingSchema(String dbName,
- String tableName, List tableCols,
- List partKeys)
- throws Exception {
+ String tableName, List tableCols,
+ List partKeys)
+ throws Exception {
String databaseName = dbName == null
- ? SqoopHCatUtilities.DEFHCATDB : dbName;
+ ? SqoopHCatUtilities.DEFHCATDB : dbName;
+ dropHCatTableIfExists(tableName, databaseName);
+ LOG.info("Successfully dropped HCatalog table if it existed previously " + databaseName
+ + '.' + tableName);
+ String createCmd = getHCatCreateTableCmd(databaseName, tableName,
+ tableCols, partKeys);
+ utils.launchHCatCli(createCmd);
+ LOG.info("Created HCatalog table " + dbName + "." + tableName);
+ }
+
+ public void dropHCatTableIfExists(String tableName, String databaseName) {
LOG.info("Dropping HCatalog table if it exists " + databaseName
- + '.' + tableName);
+ + '.' + tableName);
String dropCmd = getHCatDropTableCmd(databaseName, tableName);
try {
utils.launchHCatCli(dropCmd);
} catch (Exception e) {
LOG.debug("Drop hcatalog table exception : " + e);
- LOG.info("Unable to drop table." + dbName + "."
- + tableName + ". Assuming it did not exist");
+ LOG.info("Unable to drop table." + databaseName + "."
+ + tableName + ". Assuming it did not exist");
}
- LOG.info("Creating HCatalog table if it exists " + databaseName
- + '.' + tableName);
- String createCmd = getHCatCreateTableCmd(databaseName, tableName,
- tableCols, partKeys);
- utils.launchHCatCli(createCmd);
- LOG.info("Created HCatalog table " + dbName + "." + tableName);
}
/**