5
0
mirror of https://github.com/apache/sqoop.git synced 2025-05-02 17:22:25 +08:00

SQOOP-3134: --class-name should override default Avro schema name

This commit is contained in:
daniel voros 2019-04-05 16:15:15 +02:00 committed by Szabolcs Vasas
parent e90e244396
commit 1a04d2007d
3 changed files with 36 additions and 3 deletions

View File

@ -348,6 +348,11 @@ For example _AVRO will be converted to __AVRO.
In the case of HCatalog imports, column names are converted to lower case when In the case of HCatalog imports, column names are converted to lower case when
mapped to HCatalog columns. This may change in future. mapped to HCatalog columns. This may change in future.
During Avro imports the table's schema is saved in an +.avsc+ file under the
output directory (configured via +\--bindir+). The +\--class-name+ option can
be used to change the resulting file's name, which defaults to the table name
or in case of +\--query+ imports to +AutoGeneratedSchema+.
Incremental Imports Incremental Imports
^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^
@ -702,6 +707,9 @@ $ sqoop import --table SomeTable --jar-file mydatatypes.jar \
This command will load the +SomeTableType+ class out of +mydatatypes.jar+. This command will load the +SomeTableType+ class out of +mydatatypes.jar+.
NOTE: The +\--class-name+ option also affects the +.avsc+ file's name
generated during Avro imports.
Additional Import Configuration Properties Additional Import Configuration Properties
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
There are some additional properties which can be configured by modifying There are some additional properties which can be configured by modifying

View File

@ -104,7 +104,7 @@ public Schema generate(String schemaNameOverride) throws IOException {
} }
TableClassName tableClassName = new TableClassName(options); TableClassName tableClassName = new TableClassName(options);
String shortClassName = tableName == null ? DEFAULT_SCHEMA_NAME : tableClassName.getShortClassForTable(tableName); String shortClassName = (tableName == null && options.getClassName() == null) ? DEFAULT_SCHEMA_NAME : tableClassName.getShortClassForTable(tableName);
String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName); String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName);
String avroName = schemaNameOverride != null ? schemaNameOverride : String avroName = schemaNameOverride != null ? schemaNameOverride :
(shortClassName == null ? avroTableName : shortClassName); (shortClassName == null ? avroTableName : shortClassName);

View File

@ -38,6 +38,7 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.sqoop.testutil.ArgumentArrayBuilder;
import org.apache.sqoop.testutil.AvroTestUtils; import org.apache.sqoop.testutil.AvroTestUtils;
import org.apache.sqoop.testutil.CommonArgs; import org.apache.sqoop.testutil.CommonArgs;
import org.apache.sqoop.testutil.HsqldbTestServer; import org.apache.sqoop.testutil.HsqldbTestServer;
@ -94,6 +95,25 @@ public void testAvroImport() throws IOException {
avroImportTestHelper(null, null); avroImportTestHelper(null, null);
} }
@Test
public void testAvroFileNameWithQueryImport() throws IOException {
setCurTableName("AVRO_FILE_NAME_QUERY_IMPORT");
createTableWithColTypes(new String[] {"int"}, new String[] {"1"});
ArgumentArrayBuilder builder = new ArgumentArrayBuilder();
String[] args = builder
.withOption("connect", HsqldbTestServer.getUrl())
.withOption("query", "select * from AVRO_FILE_NAME_QUERY_IMPORT where $CONDITIONS")
.withOption("as-avrodatafile")
.withOption("m", "1")
.withOption("target-dir", getWarehouseDir() + "/AVRO_FILE_NAME_QUERY_IMPORT")
.withOption("class-name", "customAvroFile")
.build();
runImport(args);
verifySchemaFileName("customAvroFile.avsc");
}
@Test @Test
public void testDeflateCompressedAvroImport() throws IOException { public void testDeflateCompressedAvroImport() throws IOException {
this.setCurTableName("Deflate_Compressed_Avro_Import_Test_1"); this.setCurTableName("Deflate_Compressed_Avro_Import_Test_1");
@ -366,8 +386,13 @@ protected DataFileReader<GenericRecord> read(Path filename) throws IOException {
} }
protected void checkSchemaFile(final Schema schema) throws IOException { protected void checkSchemaFile(final Schema schema) throws IOException {
final File schemaFile = new File(schema.getName() + ".avsc"); String schemaFileName = schema.getName() + ".avsc";
verifySchemaFileName(schemaFileName);
assertEquals(schema, new Schema.Parser().parse(new File(schemaFileName)));
}
protected void verifySchemaFileName(String expectedFileName) {
final File schemaFile = new File(expectedFileName);
assertTrue(schemaFile.exists()); assertTrue(schemaFile.exists());
assertEquals(schema, new Schema.Parser().parse(schemaFile));
} }
} }