github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 20:40:58 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3027: Create fail fast for export with --hcatalog-table <HIVE_VIEW>, as it's not supported by HCat

Browse Source 
(Anna Szonyi via Attila Szabo)
This commit is contained in:
Attila Szabo 2016-11-23 11:39:49 +01:00
parent c08c194eb9
commit 3fba26419b
6 changed files with 270 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build.xml
View File

@ -194,11 +194,12 @@
<property name="hbase95.version" value="1.2.0" />
<property name="zookeeper.version" value="3.4.5" />
<property name="hadoop.version.full" value="2.6.0" />
<property name="hcatalog.version" value="0.13.0" />
<property name="hcatalog.version" value="1.2.0" />
<property name="hbasecompatprofile" value="2" />
<property name="avrohadoopprofile" value="2" />
</then>
</elseif>
<else>
<fail message="Unrecognized hadoopversion. Can only be 20, 23, 100, 200 or 210." />
</else>

48
src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
View File

@ -40,6 +40,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.HadoopShims.HCatHadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
@ -56,6 +59,7 @@
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.data.DefaultHCatRecord;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
@ -66,7 +70,6 @@
import org.apache.sqoop.config.ConfigurationHelper;
import org.apache.sqoop.hive.HiveTypes;
import org.apache.sqoop.manager.ConnManager;
import org.apache.sqoop.tool.BaseSqoopTool;
import org.apache.sqoop.util.Executor;
import org.apache.sqoop.util.LoggingAsyncSink;
import org.apache.sqoop.util.SubprocessSecurityManager;
@ -158,6 +161,43 @@ public final class SqoopHCatUtilities {
private static boolean testMode = false;
public static boolean isHCatView(final SqoopOptions opts) {
String hCatDatabaseName = getHCatDbNameFromOptions(opts);
String hCatTableName = getHCatTableNameFromOptions(opts);
Configuration conf = opts.getConf();
HiveConf hiveConf;
try {
if (conf != null) {
hiveConf = HCatUtil.getHiveConf(conf);
} else {
hiveConf = new HiveConf(HCatInputFormat.class);
}
HiveMetaStoreClient client = HCatUtil.getHiveClient(hiveConf);
Table table = HCatUtil.getTable(client, hCatDatabaseName, hCatTableName);
return table.isView();
}
catch(Exception e){
LOG.warn("We were not able to determine if "+hCatDatabaseName+ ":"+hCatTableName+ "is view or table.");
LOG.info("isHCatView threw an exception:", e);
return false;
}
}
private static String getHCatTableNameFromOptions(SqoopOptions opts) {
String optHCTabName = opts.getHCatTableName();
return optHCTabName.toLowerCase();
}
private static String getHCatDbNameFromOptions(SqoopOptions opts) {
String hCatDatabaseName = opts.getHCatDatabaseName() != null ? opts
.getHCatDatabaseName() : DEFHCATDB;
hCatDatabaseName = hCatDatabaseName.toLowerCase();
return hCatDatabaseName;
}
static class IntArrayWritable extends ArrayWritable {
public IntArrayWritable() {
super(IntWritable.class);
@ -278,12 +318,10 @@ public void configureHCat(final SqoopOptions opts, final Job job,
dbTableName = dbTable;
configuration = config;
hCatJob = job;
hCatDatabaseName = options.getHCatDatabaseName() != null ? options
.getHCatDatabaseName() : DEFHCATDB;
hCatDatabaseName = hCatDatabaseName.toLowerCase();
hCatDatabaseName = getHCatDbNameFromOptions(opts);
String optHCTabName = options.getHCatTableName();
hCatTableName = optHCTabName.toLowerCase();
hCatTableName = getHCatTableNameFromOptions(opts);
if (!hCatTableName.equals(optHCTabName)) {
LOG.warn("Provided HCatalog table name " + optHCTabName

8
src/java/org/apache/sqoop/tool/BaseSqoopTool.java
View File

@ -33,6 +33,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities;
import org.apache.sqoop.util.CredentialsUtil;
import org.apache.sqoop.util.LoggingUtils;
import org.apache.sqoop.util.password.CredentialProviderHelper;
@ -1560,6 +1561,9 @@ protected void validateHCatalogOptions(SqoopOptions options)
}
return;
}
if(isSet(options.getHCatTableName()) && SqoopHCatUtilities.isHCatView(options)){
throw new InvalidOptionsException("Reads/Writes from and to Views are not supported by HCatalog");
}
if (options.explicitInputDelims()) {
LOG.warn("Input field/record delimiter options are not "
@ -1672,6 +1676,10 @@ protected void validateHCatalogOptions(SqoopOptions options)
}
}
private boolean isSet(String option) {
return org.apache.commons.lang.StringUtils.isNotBlank(option);
}
protected void validateHBaseOptions(SqoopOptions options)
throws InvalidOptionsException {
if ((options.getHBaseColFamily() != null && options.getHBaseTable() == null)

92
src/test/org/apache/sqoop/hcat/HCatalogExportTest.java
View File

@ -32,8 +32,11 @@
import java.util.Map;
import java.util.TimeZone;
import com.cloudera.sqoop.SqoopOptions;
import junit.framework.JUnit4TestAdapter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
@ -46,14 +49,24 @@
import org.junit.Before;
import com.cloudera.sqoop.testutil.ExportJobTestCase;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/**
* Test that we can export HCatalog tables into databases.
*/
@RunWith(JUnit4.class)
public class HCatalogExportTest extends ExportJobTestCase {
private static final Log LOG =
LogFactory.getLog(HCatalogExportTest.class);
private HCatalogTestUtils utils = HCatalogTestUtils.instance();
@Rule
public ExpectedException exception = ExpectedException.none();
@Before
@Override
public void setUp() {
@ -142,6 +155,7 @@ protected void runHCatExport(List<String> addlArgsArray,
}
}
@Test
public void testIntTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -166,6 +180,7 @@ public void testIntTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testExportWithColumnNameValue() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -182,7 +197,7 @@ public void testExportWithColumnNameValue() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testFloatTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -201,6 +216,7 @@ public void testFloatTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testNumberTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -220,6 +236,7 @@ public void testNumberTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testDateTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -248,6 +265,7 @@ public void testDateTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testDateTypesToBigInt() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
long offset = TimeZone.getDefault().getRawOffset();
@ -270,6 +288,7 @@ public void testDateTypesToBigInt() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testStringTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -293,7 +312,7 @@ public void testStringTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testBinaryTypes() throws Exception {
ByteBuffer bb = ByteBuffer.wrap(new byte[] { 0, 1, 2 });
final int TOTAL_RECORDS = 1 * 10;
@ -310,6 +329,7 @@ public void testBinaryTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testColumnProjection() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -324,6 +344,8 @@ public void testColumnProjection() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testStaticPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -341,6 +363,7 @@ public void testStaticPartitioning() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testStaticPartitioningWithMultipleKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -361,6 +384,7 @@ public void testStaticPartitioningWithMultipleKeys() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -374,6 +398,7 @@ public void testDynamicPartitioning() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testStaticAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -394,6 +419,7 @@ public void testStaticAndDynamicPartitioning() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testMultipleStaticKeysAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -420,6 +446,7 @@ public void testMultipleStaticKeysAndDynamicPartitioning() throws Exception {
/**
* Test other file formats.
*/
@Test
public void testSequenceFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -440,6 +467,7 @@ public void testSequenceFile() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testTextFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -460,6 +488,7 @@ public void testTextFile() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
@Test
public void testPublishExportJobData() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -479,4 +508,63 @@ public void testPublishExportJobData() throws Exception {
assert (DummyDataPublisher.storeType.equals("hsqldb"));
assert (DummyDataPublisher.operation.equals("export"));
}
@Test
public void testWeCanTellIfHCatViewOrTable() throws Exception {
String tableName = getTableName().toUpperCase();
String viewName = "view";
SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
createHCatTableAndView(tableName, viewName, utils);
Configuration conf = getConf();
conf.set("oraoop.disabled", "true");
SqoopOptions opts = getSqoopOptions(conf);
opts.setHCatTableName(tableName);
assertFalse(utils.isHCatView(opts));
opts.setHCatTableName(viewName);
assertTrue(utils.isHCatView(opts));
}
@Test
public void testExportHCatViewThrowsException() throws Exception {
String tableName = getTableName().toUpperCase();
String viewName = "view";
SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
createHCatTableAndView(tableName, viewName, utils);
ArrayList<String> args = new ArrayList<String>();
args.add("--table");
args.add(getTableName());
args.add("--hcatalog-table");
args.add(viewName);
args.add("--connect");
args.add(getConnectString());
args.toArray(new String[0]);
SqoopHCatUtilities.instance().setConfigured(false);
exception.expect(java.io.IOException.class);
runExport(args.toArray(new String[0]));
}
private void createHCatTableAndView(String tableName, String viewName, SqoopHCatUtilities utils) throws Exception {
HCatalogTestUtils testUtils = HCatalogTestUtils.instance();
ColumnGenerator[] cols = new ColumnGenerator[]{
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
"boolean", Types.BOOLEAN, HCatFieldSchema.Type.BOOLEAN, 0, 0,
Boolean.TRUE, Boolean.TRUE, KeyType.NOT_A_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
"int", Types.INTEGER, HCatFieldSchema.Type.INT, 5, 5, 10,
10, KeyType.NOT_A_KEY)
};
testUtils.createHCatTable(CreateMode.CREATE_AND_LOAD, 10, tableName, cols);
String createViewCmd = "drop view " + viewName + "; create view " + viewName + " as select * from " + tableName;
utils.launchHCatCli(createViewCmd);
}
public static junit.framework.Test suite() {
return new JUnit4TestAdapter(HCatalogImportTest.class);
}
}

111
src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
View File

@ -35,6 +35,7 @@
import java.util.Map;
import java.util.TimeZone;
import junit.framework.JUnit4TestAdapter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@ -57,10 +58,16 @@
import com.cloudera.sqoop.testutil.ImportJobTestCase;
import com.cloudera.sqoop.tool.ImportTool;
import com.cloudera.sqoop.tool.SqoopTool;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/**
* Test that we can export HCatalog tables into databases.
*/
@RunWith(JUnit4.class)
public class HCatalogImportTest extends ImportJobTestCase {
private static final Log LOG =
LogFactory.getLog(HCatalogImportTest.class);
@ -68,6 +75,9 @@ public class HCatalogImportTest extends ImportJobTestCase {
private List<String> extraTestArgs = null;
private List<String> configParams = null;
@Rule
public ExpectedException exception = ExpectedException.none();
@Override
@Before
public void setUp() {
@ -343,6 +353,7 @@ protected void runHCatImport(List<String> addlArgsArray,
validateHCatRecords(recs, tblSchema, 10, cols);
}
@Test
public void testIntTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -368,6 +379,7 @@ public void testIntTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testFloatTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -387,6 +399,7 @@ public void testFloatTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testNumberTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -407,6 +420,7 @@ public void testNumberTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testDateTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -436,6 +450,7 @@ public void testDateTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testDateTypesToBigInt() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
long offset = TimeZone.getDefault().getRawOffset();
@ -459,6 +474,7 @@ public void testDateTypesToBigInt() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testStringTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -483,14 +499,16 @@ public void testStringTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testBinaryTypes() throws Exception {
ByteBuffer bb = ByteBuffer.wrap(new byte[] { 0, 1, 2 });
ByteBuffer bb10 = ByteBuffer.wrap(new byte[]{0, 1, 2, 0, 0, 0, 0, 0, 0, 0});
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
ColumnGenerator[] cols = new ColumnGenerator[] {
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
"binary(10)", Types.BINARY, HCatFieldSchema.Type.BINARY, 0, 0,
bb.array(), bb.array(), KeyType.NOT_A_KEY),
bb10.array(), bb10.array(), KeyType.NOT_A_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
"longvarbinary", Types.BINARY, HCatFieldSchema.Type.BINARY, 0, 0,
bb.array(), bb.array(), KeyType.NOT_A_KEY),
@ -500,6 +518,7 @@ public void testBinaryTypes() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testColumnProjection() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -518,6 +537,7 @@ public void testColumnProjection() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, colNames);
}
@Test
public void testColumnProjectionMissingPartKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -543,6 +563,7 @@ public void testColumnProjectionMissingPartKeys() throws Exception {
LOG.info("Exception stack trace = " + sw);
}
}
@Test
public void testStaticPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -560,6 +581,7 @@ public void testStaticPartitioning() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testStaticPartitioningWithMultipleKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -580,6 +602,7 @@ public void testStaticPartitioningWithMultipleKeys() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -593,6 +616,7 @@ public void testDynamicPartitioning() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testStaticAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -613,6 +637,7 @@ public void testStaticAndDynamicPartitioning() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testMultipleStaticKeysAndDynamicPartitioning() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -639,6 +664,7 @@ public void testMultipleStaticKeysAndDynamicPartitioning() throws Exception {
/**
* Test other file formats.
*/
@Test
public void testSequenceFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -659,6 +685,7 @@ public void testSequenceFile() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testTextFile() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -680,6 +707,7 @@ public void testTextFile() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testTableCreation() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -694,10 +722,12 @@ public void testTableCreation() throws Exception {
List<String> addlArgsArray = new ArrayList<String>();
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
null, true, false);
}
@Test
public void testTableCreationWithPartition() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -716,9 +746,11 @@ public void testTableCreationWithPartition() throws Exception {
addlArgsArray.add("2");
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
@Test
public void testTableCreationWithMultipleStaticPartKeys() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -737,9 +769,11 @@ public void testTableCreationWithMultipleStaticPartKeys() throws Exception {
addlArgsArray.add("1,2");
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
@Test
public void testTableCreationWithStorageStanza() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -760,9 +794,11 @@ public void testTableCreationWithStorageStanza() throws Exception {
addlArgsArray.add("--hcatalog-storage-stanza");
addlArgsArray.add(HCatalogTestUtils.STORED_AS_TEXT);
setExtraArgs(addlArgsArray);
utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
@Test
public void testHiveDropDelims() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -780,6 +816,7 @@ public void testHiveDropDelims() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testHiveDelimsReplacement() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -798,6 +835,7 @@ public void testHiveDelimsReplacement() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testDynamicKeyInMiddle() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -814,6 +852,7 @@ public void testDynamicKeyInMiddle() throws Exception {
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testQueryImport() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -830,6 +869,7 @@ public void testQueryImport() throws Exception {
runHCatQueryImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testCreateTableWithPreExistingTable() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -856,6 +896,7 @@ public void testCreateTableWithPreExistingTable() throws Exception {
}
}
@Test
public void testDropAndCreateWithPreExistingTable() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -875,6 +916,7 @@ public void testDropAndCreateWithPreExistingTable() throws Exception {
null, true, false);
}
@Test
public void testDropAndCreateWithoutPreExistingTable() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -893,6 +935,7 @@ public void testDropAndCreateWithoutPreExistingTable() throws Exception {
null, true, false);
}
@Test
public void testTableWithNonIdentColChars() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -905,6 +948,8 @@ public void testTableWithNonIdentColChars() throws Exception {
setExtraArgs(addlArgsArray);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
@Test
public void testTableCreationWithNonIdentColChars() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -916,10 +961,12 @@ public void testTableCreationWithNonIdentColChars() throws Exception {
List<String> addlArgsArray = new ArrayList<String>();
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
null, true, false);
}
@Test
public void testPublishQueryImportData() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -941,6 +988,7 @@ public void testPublishQueryImportData() throws Exception {
assert (DummyDataPublisher.storeTable.equals(getTableName()));
}
@Test
public void testPublishTableImportData() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
@ -959,9 +1007,70 @@ public void testPublishTableImportData() throws Exception {
List<String> addlArgsArray = new ArrayList<String>();
addlArgsArray.add("--create-hcatalog-table");
setExtraArgs(addlArgsArray);
utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
assert (DummyDataPublisher.storeType.equals("hsqldb"));
assert (DummyDataPublisher.operation.equals("import"));
assert (DummyDataPublisher.storeTable.equals(getTableName()));
}
@Test
public void testWeCanTellIfHCatViewOrTable() throws Exception {
String tableName = getTableName().toUpperCase();
String viewName = "view";
SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
createHCatTableAndView(tableName, viewName, utils);
Configuration conf = getConf();
conf.set("oraoop.disabled", "true");
SqoopOptions opts = getSqoopOptions(conf);
opts.setHCatTableName(tableName);
assertFalse(utils.isHCatView(opts));
opts.setHCatTableName(viewName);
assertTrue(utils.isHCatView(opts));
}
@Test
public void testImportHCatViewThrowsException() throws Exception {
String tableName = getTableName().toUpperCase();
String viewName = "view";
SqoopHCatUtilities utils = SqoopHCatUtilities.instance();
createHCatTableAndView(tableName, viewName, utils);
ArrayList<String> args = new ArrayList<String>();
args.addAll(getConfigParams());
args.add("--table");
args.add(getTableName());
args.add("--hcatalog-table");
args.add(viewName);
args.add("--connect");
args.add(getConnectString());
args.toArray(new String[0]);
SqoopHCatUtilities.instance().setConfigured(false);
exception.expect(java.io.IOException.class);
runImport(new ImportTool(), args.toArray(new String[0]));
}
private void createHCatTableAndView(String tableName, String viewName, SqoopHCatUtilities utils) throws Exception {
HCatalogTestUtils testUtils = HCatalogTestUtils.instance();
ColumnGenerator[] cols = new ColumnGenerator[]{
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
"boolean", Types.BOOLEAN, HCatFieldSchema.Type.BOOLEAN, 0, 0,
Boolean.TRUE, Boolean.TRUE, KeyType.NOT_A_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
"int", Types.INTEGER, HCatFieldSchema.Type.INT, 5, 5, 10,
10, KeyType.NOT_A_KEY)
};
testUtils.createHCatTable(CreateMode.CREATE_AND_LOAD, 10, tableName, cols);
String createViewCmd = "drop view " + viewName + "; create view " + viewName + " as select * from " + tableName;
utils.launchHCatCli(createViewCmd);
}
public static junit.framework.Test suite() {
return new JUnit4TestAdapter(HCatalogImportTest.class);
}
}

30
src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
View File

@ -177,29 +177,33 @@ private static String getHCatCreateTableCmd(String dbName,
* memory list.
*/
public void createHCatTableUsingSchema(String dbName,
String tableName, List<HCatFieldSchema> tableCols,
List<HCatFieldSchema> partKeys)
throws Exception {
String tableName, List<HCatFieldSchema> tableCols,
List<HCatFieldSchema> partKeys)
throws Exception {
String databaseName = dbName == null
? SqoopHCatUtilities.DEFHCATDB : dbName;
? SqoopHCatUtilities.DEFHCATDB : dbName;
dropHCatTableIfExists(tableName, databaseName);
LOG.info("Successfully dropped HCatalog table if it existed previously " + databaseName
+ '.' + tableName);
String createCmd = getHCatCreateTableCmd(databaseName, tableName,
tableCols, partKeys);
utils.launchHCatCli(createCmd);
LOG.info("Created HCatalog table " + dbName + "." + tableName);
}
public void dropHCatTableIfExists(String tableName, String databaseName) {
LOG.info("Dropping HCatalog table if it exists " + databaseName
+ '.' + tableName);
+ '.' + tableName);
String dropCmd = getHCatDropTableCmd(databaseName, tableName);
try {
utils.launchHCatCli(dropCmd);
} catch (Exception e) {
LOG.debug("Drop hcatalog table exception : " + e);
LOG.info("Unable to drop table." + dbName + "."
+ tableName + ". Assuming it did not exist");
LOG.info("Unable to drop table." + databaseName + "."
+ tableName + ". Assuming it did not exist");
}
LOG.info("Creating HCatalog table if it exists " + databaseName
+ '.' + tableName);
String createCmd = getHCatCreateTableCmd(databaseName, tableName,
tableCols, partKeys);
utils.launchHCatCli(createCmd);
LOG.info("Created HCatalog table " + dbName + "." + tableName);
}
/**