github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-02 17:22:25 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3134: --class-name should override default Avro schema name

Browse Source
This commit is contained in:
daniel voros 2019-04-05 16:15:15 +02:00 committed by Szabolcs Vasas
parent e90e244396
commit 1a04d2007d
3 changed files with 36 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/docs/user/import.txt
View File

@ -348,6 +348,11 @@ For example _AVRO will be converted to __AVRO.
In the case of HCatalog imports, column names are converted to lower case when
mapped to HCatalog columns. This may change in future.
During Avro imports the table's schema is saved in an +.avsc+ file under the
output directory (configured via +\--bindir+). The +\--class-name+ option can
be used to change the resulting file's name, which defaults to the table name
or in case of +\--query+ imports to +AutoGeneratedSchema+.
Incremental Imports
^^^^^^^^^^^^^^^^^^^
@ -702,6 +707,9 @@ $ sqoop import --table SomeTable --jar-file mydatatypes.jar \
This command will load the +SomeTableType+ class out of +mydatatypes.jar+.
NOTE: The +\--class-name+ option also affects the +.avsc+ file's name
generated during Avro imports.
Additional Import Configuration Properties
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
There are some additional properties which can be configured by modifying

2
src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
View File

@ -104,7 +104,7 @@ public Schema generate(String schemaNameOverride) throws IOException {
}
TableClassName tableClassName = new TableClassName(options);
String shortClassName = tableName == null ? DEFAULT_SCHEMA_NAME : tableClassName.getShortClassForTable(tableName);
String shortClassName = (tableName == null && options.getClassName() == null) ? DEFAULT_SCHEMA_NAME : tableClassName.getShortClassForTable(tableName);
String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName);
String avroName = schemaNameOverride != null ? schemaNameOverride :
(shortClassName == null ? avroTableName : shortClassName);

29
src/test/org/apache/sqoop/TestAvroImport.java
View File

@ -38,6 +38,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.sqoop.testutil.ArgumentArrayBuilder;
import org.apache.sqoop.testutil.AvroTestUtils;
import org.apache.sqoop.testutil.CommonArgs;
import org.apache.sqoop.testutil.HsqldbTestServer;
@ -94,6 +95,25 @@ public void testAvroImport() throws IOException {
avroImportTestHelper(null, null);
}
@Test
public void testAvroFileNameWithQueryImport() throws IOException {
setCurTableName("AVRO_FILE_NAME_QUERY_IMPORT");
createTableWithColTypes(new String[] {"int"}, new String[] {"1"});
ArgumentArrayBuilder builder = new ArgumentArrayBuilder();
String[] args = builder
.withOption("connect", HsqldbTestServer.getUrl())
.withOption("query", "select * from AVRO_FILE_NAME_QUERY_IMPORT where $CONDITIONS")
.withOption("as-avrodatafile")
.withOption("m", "1")
.withOption("target-dir", getWarehouseDir() + "/AVRO_FILE_NAME_QUERY_IMPORT")
.withOption("class-name", "customAvroFile")
.build();
runImport(args);
verifySchemaFileName("customAvroFile.avsc");
}
@Test
public void testDeflateCompressedAvroImport() throws IOException {
this.setCurTableName("Deflate_Compressed_Avro_Import_Test_1");
@ -366,8 +386,13 @@ protected DataFileReader<GenericRecord> read(Path filename) throws IOException {
}
protected void checkSchemaFile(final Schema schema) throws IOException {
final File schemaFile = new File(schema.getName() + ".avsc");
String schemaFileName = schema.getName() + ".avsc";
verifySchemaFileName(schemaFileName);
assertEquals(schema, new Schema.Parser().parse(new File(schemaFileName)));
}
protected void verifySchemaFileName(String expectedFileName) {
final File schemaFile = new File(expectedFileName);
assertTrue(schemaFile.exists());
assertEquals(schema, new Schema.Parser().parse(schemaFile));
}
}