diff --git a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java index e70d23c6..d5bfae27 100644 --- a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java +++ b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java @@ -23,8 +23,6 @@ import java.sql.SQLException; import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -88,7 +86,8 @@ protected void configureMapper(Job job, String tableName, job.setOutputValueClass(NullWritable.class); } else if (options.getFileLayout() == SqoopOptions.FileLayout.AvroDataFile) { - Schema schema = generateAvroSchema(tableName); + final String schemaNameOverride = null; + Schema schema = generateAvroSchema(tableName, schemaNameOverride); try { writeAvroSchema(schema); } catch (final IOException e) { @@ -99,9 +98,12 @@ protected void configureMapper(Job job, String tableName, } else if (options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) { Configuration conf = job.getConfiguration(); - // An Avro schema is required for creating a dataset that manages - // Parquet data records. The import will fail, if schema is invalid. - Schema schema = generateAvroSchema(tableName); + // Kite SDK requires an Avro schema to represent the data structure of + // target dataset. If the schema name equals to generated java class name, + // the import will fail. So we use table name as schema name and add a + // prefix "codegen_" to generated java class to avoid the conflict. + final String schemaNameOverride = tableName; + Schema schema = generateAvroSchema(tableName, schemaNameOverride); String uri = getKiteUri(conf, tableName); ParquetJob.configureImportJob(conf, schema, uri, options.isAppendMode(), options.doHiveImport() && options.doOverwriteHiveTable()); @@ -123,11 +125,12 @@ private String getKiteUri(Configuration conf, String tableName) throws IOExcepti } } - private Schema generateAvroSchema(String tableName) throws IOException { + private Schema generateAvroSchema(String tableName, + String schemaNameOverride) throws IOException { ConnManager connManager = getContext().getConnManager(); AvroSchemaGenerator generator = new AvroSchemaGenerator(options, connManager, tableName); - return generator.generate(); + return generator.generate(schemaNameOverride); } private void writeAvroSchema(final Schema schema) throws IOException { diff --git a/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java b/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java index ed8e8b11..2576673b 100644 --- a/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java +++ b/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java @@ -55,7 +55,7 @@ public AvroSchemaGenerator(final SqoopOptions opts, final ConnManager connMgr, this.tableName = table; } - public Schema generate() throws IOException { + public Schema generate(String schemaNameOverride) throws IOException { ClassWriter classWriter = new ClassWriter(options, connManager, tableName, null); Map columnTypes = classWriter.getColumnTypes(); @@ -75,7 +75,8 @@ public Schema generate() throws IOException { TableClassName tableClassName = new TableClassName(options); String shortClassName = tableClassName.getShortClassForTable(tableName); String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName); - String avroName = "sqoop_import_" + (shortClassName == null ? avroTableName : shortClassName); + String avroName = schemaNameOverride != null ? schemaNameOverride : + (shortClassName == null ? avroTableName : shortClassName); String avroNamespace = tableClassName.getPackageForTable(); String doc = "Sqoop import of " + avroTableName; diff --git a/src/java/org/apache/sqoop/tool/CodeGenTool.java b/src/java/org/apache/sqoop/tool/CodeGenTool.java index 6bd7f1d5..ab339adf 100644 --- a/src/java/org/apache/sqoop/tool/CodeGenTool.java +++ b/src/java/org/apache/sqoop/tool/CodeGenTool.java @@ -90,6 +90,17 @@ public String generateORM(SqoopOptions options, String tableName) return null; } LOG.info("Beginning code generation"); + + if (options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) { + String className = options.getClassName() != null ? + options.getClassName() : options.getTableName(); + if (className.equalsIgnoreCase(options.getTableName())) { + className = "codegen_" + className; + options.setClassName(className); + LOG.info("Will generate java class as " + options.getClassName()); + } + } + CompilationManager compileMgr = new CompilationManager(options); ClassWriter classWriter = new ClassWriter(options, manager, tableName, compileMgr);