diff --git a/src/java/org/apache/sqoop/manager/ConnManager.java b/src/java/org/apache/sqoop/manager/ConnManager.java index 1b32dc9c..a1ac38e7 100644 --- a/src/java/org/apache/sqoop/manager/ConnManager.java +++ b/src/java/org/apache/sqoop/manager/ConnManager.java @@ -639,5 +639,20 @@ public String datetimeToQueryString(String datetime, int columnType) { public String getInputBoundsQuery(String splitByCol, String sanitizedQuery) { return null; } + + /** + * This method allows the ConnManager to override the generation of ORM + * classes if the SQOOP generated classes are not used by it. + * A return value of false from this method means that the SQOOP ORM + * classes are needed to use with the connector. + * A return value of true indicates that the connection manager does not + * use the SQOOP ORM classes. For example, in the Direct mode of some of + * the connectors, the text files are directly processed by DB specific + * facilities without even being passed through the SQOOP process and + * in those circumstances, it makes sense to disable the ORM generation. + */ + public boolean isORMFacilitySelfManaged() { + return false; + } } diff --git a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java index 0a1e6056..ef989365 100644 --- a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java +++ b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java @@ -246,4 +246,9 @@ private void handleNetezzaExtraArgs(SqoopOptions opts) public boolean supportsStagingForExport() { return false; } + + @Override + public boolean isORMFacilitySelfManaged() { + return true; + } } diff --git a/src/java/org/apache/sqoop/manager/ExportJobContext.java b/src/java/org/apache/sqoop/manager/ExportJobContext.java index 5699e2fb..2a6f2b59 100644 --- a/src/java/org/apache/sqoop/manager/ExportJobContext.java +++ b/src/java/org/apache/sqoop/manager/ExportJobContext.java @@ -18,6 +18,9 @@ package org.apache.sqoop.manager; +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.util.Jars; + import com.cloudera.sqoop.SqoopOptions; /** @@ -35,6 +38,10 @@ public ExportJobContext(final String table, final String jar, final SqoopOptions opts) { this.tableName = table; this.jarFile = jar; + if (this.jarFile == null) { + // Set the jarFile to the hadoop core jar file. + this.jarFile = Jars.getJarPathForClass(Configuration.class); + } this.options = opts; } diff --git a/src/java/org/apache/sqoop/manager/ImportJobContext.java b/src/java/org/apache/sqoop/manager/ImportJobContext.java index 09a7abe2..354cd15e 100644 --- a/src/java/org/apache/sqoop/manager/ImportJobContext.java +++ b/src/java/org/apache/sqoop/manager/ImportJobContext.java @@ -21,7 +21,10 @@ import org.apache.hadoop.mapreduce.InputFormat; import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat; import com.cloudera.sqoop.SqoopOptions; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.sqoop.util.Jars; /** * A set of parameters describing an import operation; this is passed to @@ -40,6 +43,10 @@ public ImportJobContext(final String table, final String jar, final SqoopOptions opts, final Path destination) { this.tableName = table; this.jarFile = jar; + if (this.jarFile == null) { + // Set the jarFile to the hadoop core jar file. + this.jarFile = Jars.getJarPathForClass(Configuration.class); + } this.options = opts; this.inputFormatClass = DataDrivenDBInputFormat.class; this.destination = destination; diff --git a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java index ff849747..1065d0ba 100644 --- a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java +++ b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java @@ -321,8 +321,14 @@ public void runExport() throws ExportException, IOException { } } - String tableClassName = - new TableClassName(options).getClassForTable(outputTableName); + + String tableClassName = null; + if (!cmgr.isORMFacilitySelfManaged()) { + tableClassName = + new TableClassName(options).getClassForTable(outputTableName); + } + // For ORM self managed, we leave the tableClassName to null so that + // we don't check for non-existing classes. String ormJarFile = context.getJarFile(); LOG.info("Beginning export of " + outputTableName); diff --git a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java index f766532e..2465f3f4 100644 --- a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java +++ b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java @@ -196,9 +196,14 @@ public void runImport(String tableName, String ormJarFile, String splitByCol, } else { LOG.info("Beginning query import."); } + String tableClassName = null; + if (!getContext().getConnManager().isORMFacilitySelfManaged()) { + tableClassName = + new TableClassName(options).getClassForTable(tableName); + } + // For ORM self managed, we leave the tableClassName to null so that + // we don't check for non-existing classes. - String tableClassName = - new TableClassName(options).getClassForTable(tableName); loadJars(conf, ormJarFile, tableClassName); Job job = new Job(conf); diff --git a/src/java/org/apache/sqoop/mapreduce/JobBase.java b/src/java/org/apache/sqoop/mapreduce/JobBase.java index 4e7723fd..0df11562 100644 --- a/src/java/org/apache/sqoop/mapreduce/JobBase.java +++ b/src/java/org/apache/sqoop/mapreduce/JobBase.java @@ -220,6 +220,7 @@ private void addDirToCache(File dir, FileSystem fs, Set localUrls) { */ protected void loadJars(Configuration conf, String ormJarFile, String tableClassName) throws IOException { + boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address")) || "local".equals(conf.get("mapred.job.tracker")); if (isLocal) { diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java index 982e4441..0202f7f4 100644 --- a/src/java/org/apache/sqoop/orm/ClassWriter.java +++ b/src/java/org/apache/sqoop/orm/ClassWriter.java @@ -1064,13 +1064,24 @@ private void generateHadoopWrite(Map columnTypes, return cleanedColNames; } + /** + * Made this a separate method to overcome the 150 line limit of checkstyle. + */ + private void logORMSelfGenerationMessage() { + LOG.info("The connection manager declares that it self manages mapping" + + " between records & fields and rows & columns. No class will" + + " will be generated."); + } /** * Generate the ORM code for the class. */ public void generate() throws IOException { Map columnTypes = getColumnTypes(); - + if (connManager.isORMFacilitySelfManaged()) { + logORMSelfGenerationMessage(); + return; + } if (columnTypes == null) { throw new IOException("No columns to generate for ClassWriter"); } @@ -1110,7 +1121,6 @@ public void generate() throws IOException { } columnTypes.put(identifier, type); } - // Check that all explicitly mapped columns are present in result set Properties mapping = options.getMapColumnJava(); if (mapping != null && !mapping.isEmpty()) { @@ -1207,7 +1217,6 @@ public void generate() throws IOException { // ignored because we're closing. } } - if (null != ostream) { try { ostream.close(); diff --git a/src/java/org/apache/sqoop/tool/CodeGenTool.java b/src/java/org/apache/sqoop/tool/CodeGenTool.java index 8a4aa426..dd34a97d 100644 --- a/src/java/org/apache/sqoop/tool/CodeGenTool.java +++ b/src/java/org/apache/sqoop/tool/CodeGenTool.java @@ -71,10 +71,24 @@ public String generateORM(SqoopOptions options, String tableName) // This code generator is being invoked as part of an import or export // process, and the user has pre-specified a jar and class to use. // Don't generate. + if (manager.isORMFacilitySelfManaged()) { + // No need to generated any ORM. Ignore any jar file given on + // command line also. + LOG.info("The connection manager declares that it self manages mapping" + + " between records & fields and rows & columns. The jar file " + + " provided will have no effect"); + } LOG.info("Using existing jar: " + existingJar); return existingJar; } - + if (manager.isORMFacilitySelfManaged()) { + // No need to generated any ORM. Ignore any jar file given on + // command line also. + LOG.info("The connection manager declares that it self manages mapping" + + " between records & fields and rows & columns. No class will" + + " will be generated."); + return null; + } LOG.info("Beginning code generation"); CompilationManager compileMgr = new CompilationManager(options); ClassWriter classWriter = new ClassWriter(options, manager, tableName, diff --git a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java index 3b775717..bfb25b07 100644 --- a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java +++ b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java @@ -38,6 +38,7 @@ import org.junit.Test; import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.TestConnFactory.DummyManager; import com.cloudera.sqoop.manager.ConnManager; import com.cloudera.sqoop.testutil.DirUtil; import com.cloudera.sqoop.testutil.HsqldbTestServer; @@ -485,4 +486,51 @@ public void testBrokenUserMapping() throws Exception { } fail("we shouldn't successfully generate code"); } + + private void runFailedGenerationTest(String [] argv, + String classNameToCheck) { + File codeGenDirFile = new File(CODE_GEN_DIR); + File classGenDirFile = new File(JAR_GEN_DIR); + + try { + options = new ImportTool().parseArguments(argv, + null, options, true); + } catch (Exception e) { + LOG.error("Could not parse options: " + e.toString()); + } + + CompilationManager compileMgr = new CompilationManager(options); + ClassWriter writer = new ClassWriter(options, manager, + HsqldbTestServer.getTableName(), compileMgr); + + try { + writer.generate(); + compileMgr.compile(); + fail("ORM class file generation succeeded when it was expected to fail"); + } catch (IOException ioe) { + LOG.error("Got IOException from ORM generation as expected : " + + ioe.toString()); + } + } + /** + * A dummy manager that declares that it ORM is self managed. + */ + public static class DummyDirectManager extends DummyManager { + @Override + public boolean isORMFacilitySelfManaged() { + return true; + } + } + + @Test + public void testNoClassGeneration() throws Exception { + manager = new DummyDirectManager(); + String [] argv = { + "--bindir", + JAR_GEN_DIR, + "--outdir", + CODE_GEN_DIR, + }; + runFailedGenerationTest(argv, HsqldbTestServer.getTableName()); + } }