mirror of
https://github.com/apache/sqoop.git
synced 2025-05-02 17:22:25 +08:00
SQOOP-3134: --class-name should override default Avro schema name
This commit is contained in:
parent
e90e244396
commit
1a04d2007d
@ -348,6 +348,11 @@ For example _AVRO will be converted to __AVRO.
|
|||||||
In the case of HCatalog imports, column names are converted to lower case when
|
In the case of HCatalog imports, column names are converted to lower case when
|
||||||
mapped to HCatalog columns. This may change in future.
|
mapped to HCatalog columns. This may change in future.
|
||||||
|
|
||||||
|
During Avro imports the table's schema is saved in an +.avsc+ file under the
|
||||||
|
output directory (configured via +\--bindir+). The +\--class-name+ option can
|
||||||
|
be used to change the resulting file's name, which defaults to the table name
|
||||||
|
or in case of +\--query+ imports to +AutoGeneratedSchema+.
|
||||||
|
|
||||||
Incremental Imports
|
Incremental Imports
|
||||||
^^^^^^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
@ -702,6 +707,9 @@ $ sqoop import --table SomeTable --jar-file mydatatypes.jar \
|
|||||||
|
|
||||||
This command will load the +SomeTableType+ class out of +mydatatypes.jar+.
|
This command will load the +SomeTableType+ class out of +mydatatypes.jar+.
|
||||||
|
|
||||||
|
NOTE: The +\--class-name+ option also affects the +.avsc+ file's name
|
||||||
|
generated during Avro imports.
|
||||||
|
|
||||||
Additional Import Configuration Properties
|
Additional Import Configuration Properties
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
There are some additional properties which can be configured by modifying
|
There are some additional properties which can be configured by modifying
|
||||||
|
@ -104,7 +104,7 @@ public Schema generate(String schemaNameOverride) throws IOException {
|
|||||||
}
|
}
|
||||||
|
|
||||||
TableClassName tableClassName = new TableClassName(options);
|
TableClassName tableClassName = new TableClassName(options);
|
||||||
String shortClassName = tableName == null ? DEFAULT_SCHEMA_NAME : tableClassName.getShortClassForTable(tableName);
|
String shortClassName = (tableName == null && options.getClassName() == null) ? DEFAULT_SCHEMA_NAME : tableClassName.getShortClassForTable(tableName);
|
||||||
String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName);
|
String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName);
|
||||||
String avroName = schemaNameOverride != null ? schemaNameOverride :
|
String avroName = schemaNameOverride != null ? schemaNameOverride :
|
||||||
(shortClassName == null ? avroTableName : shortClassName);
|
(shortClassName == null ? avroTableName : shortClassName);
|
||||||
|
@ -38,6 +38,7 @@
|
|||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
|
|
||||||
|
import org.apache.sqoop.testutil.ArgumentArrayBuilder;
|
||||||
import org.apache.sqoop.testutil.AvroTestUtils;
|
import org.apache.sqoop.testutil.AvroTestUtils;
|
||||||
import org.apache.sqoop.testutil.CommonArgs;
|
import org.apache.sqoop.testutil.CommonArgs;
|
||||||
import org.apache.sqoop.testutil.HsqldbTestServer;
|
import org.apache.sqoop.testutil.HsqldbTestServer;
|
||||||
@ -94,6 +95,25 @@ public void testAvroImport() throws IOException {
|
|||||||
avroImportTestHelper(null, null);
|
avroImportTestHelper(null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testAvroFileNameWithQueryImport() throws IOException {
|
||||||
|
setCurTableName("AVRO_FILE_NAME_QUERY_IMPORT");
|
||||||
|
createTableWithColTypes(new String[] {"int"}, new String[] {"1"});
|
||||||
|
ArgumentArrayBuilder builder = new ArgumentArrayBuilder();
|
||||||
|
String[] args = builder
|
||||||
|
.withOption("connect", HsqldbTestServer.getUrl())
|
||||||
|
.withOption("query", "select * from AVRO_FILE_NAME_QUERY_IMPORT where $CONDITIONS")
|
||||||
|
.withOption("as-avrodatafile")
|
||||||
|
.withOption("m", "1")
|
||||||
|
.withOption("target-dir", getWarehouseDir() + "/AVRO_FILE_NAME_QUERY_IMPORT")
|
||||||
|
.withOption("class-name", "customAvroFile")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
runImport(args);
|
||||||
|
|
||||||
|
verifySchemaFileName("customAvroFile.avsc");
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testDeflateCompressedAvroImport() throws IOException {
|
public void testDeflateCompressedAvroImport() throws IOException {
|
||||||
this.setCurTableName("Deflate_Compressed_Avro_Import_Test_1");
|
this.setCurTableName("Deflate_Compressed_Avro_Import_Test_1");
|
||||||
@ -366,8 +386,13 @@ protected DataFileReader<GenericRecord> read(Path filename) throws IOException {
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected void checkSchemaFile(final Schema schema) throws IOException {
|
protected void checkSchemaFile(final Schema schema) throws IOException {
|
||||||
final File schemaFile = new File(schema.getName() + ".avsc");
|
String schemaFileName = schema.getName() + ".avsc";
|
||||||
|
verifySchemaFileName(schemaFileName);
|
||||||
|
assertEquals(schema, new Schema.Parser().parse(new File(schemaFileName)));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void verifySchemaFileName(String expectedFileName) {
|
||||||
|
final File schemaFile = new File(expectedFileName);
|
||||||
assertTrue(schemaFile.exists());
|
assertTrue(schemaFile.exists());
|
||||||
assertEquals(schema, new Schema.Parser().parse(schemaFile));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user