mirror of
https://github.com/apache/sqoop.git
synced 2025-05-03 20:40:58 +08:00
SQOOP-3010: Sqoop should not allow --as-parquetfile with hcatalog jobs or when hive import with create-hive-table is used
(Sowmya Ramesh via Venkat Ranganathan)
This commit is contained in:
parent
b007e4d59d
commit
8f3989d9ad
@ -1421,6 +1421,14 @@ protected void validateHiveOptions(SqoopOptions options)
|
|||||||
+ "importing into SequenceFile format.");
|
+ "importing into SequenceFile format.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Hive import and create hive table not compatible for ParquetFile format
|
||||||
|
if (options.doHiveImport()
|
||||||
|
&& options.doFailIfHiveTableExists()
|
||||||
|
&& options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) {
|
||||||
|
throw new InvalidOptionsException("Hive import and create hive table is not compatible with "
|
||||||
|
+ "importing into ParquetFile format.");
|
||||||
|
}
|
||||||
|
|
||||||
if (options.doHiveImport()
|
if (options.doHiveImport()
|
||||||
&& options.isAppendMode()
|
&& options.isAppendMode()
|
||||||
&& !options.getIncrementalMode().equals(IncrementalMode.AppendRows)) {
|
&& !options.getIncrementalMode().equals(IncrementalMode.AppendRows)) {
|
||||||
@ -1598,6 +1606,12 @@ protected void validateHCatalogOptions(SqoopOptions options)
|
|||||||
+ " option." + HELP_STR);
|
+ " option." + HELP_STR);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) {
|
||||||
|
throw new InvalidOptionsException("HCatalog job is not compatible with "
|
||||||
|
+ "SequenceFile format option " + FMT_PARQUETFILE_ARG
|
||||||
|
+ " option." + HELP_STR);
|
||||||
|
}
|
||||||
|
|
||||||
if (options.getHCatalogPartitionKeys() != null
|
if (options.getHCatalogPartitionKeys() != null
|
||||||
&& options.getHCatalogPartitionValues() == null) {
|
&& options.getHCatalogPartitionValues() == null) {
|
||||||
throw new InvalidOptionsException("Either both --hcatalog-partition-keys"
|
throw new InvalidOptionsException("Either both --hcatalog-partition-keys"
|
||||||
|
@ -402,6 +402,27 @@ public void testAppendHiveImportAsParquet() throws IOException {
|
|||||||
{"test2", 4242, "somestring2"}, {"test", 42, "somestring"}});
|
{"test2", 4242, "somestring2"}, {"test", 42, "somestring"}});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test hive create and --as-parquetfile options validation.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testCreateHiveImportAsParquet() throws ParseException {
|
||||||
|
final String TABLE_NAME = "CREATE_HIVE_IMPORT_AS_PARQUET";
|
||||||
|
setCurTableName(TABLE_NAME);
|
||||||
|
setNumCols(3);
|
||||||
|
String [] extraArgs = {"--as-parquetfile", "--create-hive-table"};
|
||||||
|
ImportTool tool = new ImportTool();
|
||||||
|
|
||||||
|
try {
|
||||||
|
tool.validateOptions(tool.parseArguments(getArgv(false, extraArgs), null,
|
||||||
|
null, true));
|
||||||
|
fail("Expected InvalidOptionsException");
|
||||||
|
} catch (InvalidOptionsException ex) {
|
||||||
|
/* success */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/** Test that dates are coerced properly to strings. */
|
/** Test that dates are coerced properly to strings. */
|
||||||
@Test
|
@Test
|
||||||
public void testDate() throws IOException {
|
public void testDate() throws IOException {
|
||||||
|
@ -164,6 +164,24 @@ public void testHCatExportWithExportDir() throws Exception {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testHCatExportWithParquetFile() throws Exception {
|
||||||
|
String[] args = {
|
||||||
|
"--connect",
|
||||||
|
"jdbc:db:url",
|
||||||
|
"--table",
|
||||||
|
"dbtable",
|
||||||
|
"--hcatalog-table",
|
||||||
|
"table",
|
||||||
|
"--as-parquetfile",
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
SqoopOptions opts = parseExportArgs(args);
|
||||||
|
exportTool.validateOptions(opts);
|
||||||
|
fail("Expected InvalidOptionsException");
|
||||||
|
} catch (SqoopOptions.InvalidOptionsException ioe) {
|
||||||
|
// expected.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void testHCatImportWithSequenceFile() throws Exception {
|
public void testHCatImportWithSequenceFile() throws Exception {
|
||||||
String[] args = {
|
String[] args = {
|
||||||
@ -184,6 +202,28 @@ public void testHCatImportWithSequenceFile() throws Exception {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testHCatImportWithParquetFile() throws Exception {
|
||||||
|
String[] args = {
|
||||||
|
"--hcatalog-table",
|
||||||
|
"table",
|
||||||
|
"--create-hcatalog-table",
|
||||||
|
"--connect",
|
||||||
|
"jdbc:db:url",
|
||||||
|
"--table",
|
||||||
|
"dbtable",
|
||||||
|
"--hcatalog-table",
|
||||||
|
"table",
|
||||||
|
"--as-parquetfile",
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
SqoopOptions opts = parseImportArgs(args);
|
||||||
|
importTool.validateOptions(opts);
|
||||||
|
fail("Expected InvalidOptionsException");
|
||||||
|
} catch (SqoopOptions.InvalidOptionsException ioe) {
|
||||||
|
// expected.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void testHCatImportWithAvroFile() throws Exception {
|
public void testHCatImportWithAvroFile() throws Exception {
|
||||||
String[] args = {
|
String[] args = {
|
||||||
"--connect",
|
"--connect",
|
||||||
|
Loading…
Reference in New Issue
Block a user