diff --git a/src/docs/man/codegen-args.txt b/src/docs/man/codegen-args.txt index c1afaef9..adee7d9e 100644 --- a/src/docs/man/codegen-args.txt +++ b/src/docs/man/codegen-args.txt @@ -34,6 +34,9 @@ Code generation options --outdir (dir):: Output directory for generated code +--delete-compile-dir:: + Remove temporarily generated class Jar files after job finishes + --package-name (package):: Puts auto-generated classes in the named Java package diff --git a/src/docs/user/common-args.txt b/src/docs/user/common-args.txt index 98f19be0..6d7560a3 100644 --- a/src/docs/user/common-args.txt +++ b/src/docs/user/common-args.txt @@ -34,6 +34,8 @@ Argument Description +-P+ Read password from console +\--password + Set authentication password +\--username + Set authentication username ++\--delete-compile-dir+ Remove temporarily generated class Jar\ + files after job finishes +\--verbose+ Print more information while working +\--connection-param-file + Optional properties file that\ provides connection parameters diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index 3a19aeac..cc1b7528 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -204,6 +204,7 @@ public String toString() { @StoredAsProperty("codegen.output.dir") private String codeOutputDir; @StoredAsProperty("codegen.compile.dir") private String jarOutputDir; + @StoredAsProperty("codegen.delete.compile.dir") private boolean deleteJarOutputDir; // Boolean specifying whether jarOutputDir is a nonce tmpdir (true), or // explicitly set by the user (false). If the former, disregard any value // for jarOutputDir saved in the metastore. @@ -1086,6 +1087,8 @@ private void initDefaults(Configuration baseConfiguration) { this.jarOutputDir = getNonceJarDir(tmpDir + "sqoop-" + localUsername + "/compile"); this.jarDirIsAuto = true; + // Default to false, so behaviour does not change + this.deleteJarOutputDir = false; this.layout = FileLayout.TextFile; this.areOutputDelimsManuallySet = false; @@ -1706,6 +1709,17 @@ public void setJarOutputDir(String outDir) { this.jarDirIsAuto = false; } + /** + * @return boolean - whether or not to delete the compile JAR directory + */ + public Boolean getDeleteJarOutputDir() { + return this.deleteJarOutputDir; + } + + public void setDeleteJarOutputDir(Boolean delete) { + this.deleteJarOutputDir = delete; + } + /** * Return the value of $HADOOP_MAPRED_HOME. * @return $HADOOP_MAPRED_HOME, or null if it's not set. diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java index a4a768af..46d0698b 100644 --- a/src/java/org/apache/sqoop/orm/ClassWriter.java +++ b/src/java/org/apache/sqoop/orm/ClassWriter.java @@ -51,6 +51,7 @@ import org.apache.sqoop.lib.RecordParser; import org.apache.sqoop.lib.SqoopRecord; import org.apache.sqoop.manager.ConnManager; +import org.apache.sqoop.util.DirCleanupHook; /** * Creates an ORM class to represent a table from a database. @@ -1785,6 +1786,12 @@ public void generate() throws IOException { LOG.debug("sourceFilename is " + sourceFilename); } + // SQOOP-3042 - Sqoop does not clear compile directory under /tmp/sqoop-/compile + // Add shutdown hook so that all files under the jar output dir can be cleared + if(options.getDeleteJarOutputDir()) { + Runtime.getRuntime().addShutdownHook(new DirCleanupHook(options.getJarOutputDir())); + } + compileManager.addSourceFile(sourceFilename); // Create any missing parent directories. diff --git a/src/java/org/apache/sqoop/orm/CompilationManager.java b/src/java/org/apache/sqoop/orm/CompilationManager.java index 6590cacc..f0f5de74 100644 --- a/src/java/org/apache/sqoop/orm/CompilationManager.java +++ b/src/java/org/apache/sqoop/orm/CompilationManager.java @@ -245,13 +245,14 @@ public void compile() throws IOException { } } try { - FileUtils.moveFile(fOrig, fDest); + LOG.debug("moving file from " + fOrig.getAbsolutePath() + " to " + fDest.getAbsolutePath()); + FileUtils.moveFile(fOrig, fDest); } catch (IOException e) { - /*Removed the exception being thrown - *even if the .java file can not be renamed - *or can not be moved a "dest" directory for - *any reason.*/ - LOG.debug("Could not rename " + orig + " to " + dest); + /*Removed the exception being thrown + *even if the .java file can not be renamed + *or can not be moved a "dest" directory for + *any reason.*/ + LOG.error("Could not rename " + orig + " to " + dest + ". Error: " + e.getMessage()); } } } diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 8d318327..e505c267 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -102,6 +102,7 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool { public static final String TARGET_DIR_ARG = "target-dir"; public static final String APPEND_ARG = "append"; public static final String DELETE_ARG = "delete-target-dir"; + public static final String DELETE_COMPILE_ARG = "delete-compile-dir"; public static final String NULL_STRING = "null-string"; public static final String INPUT_NULL_STRING = "input-null-string"; public static final String NULL_NON_STRING = "null-non-string"; @@ -805,6 +806,10 @@ protected RelatedOptions getCodeGenOpts(boolean multiTable) { .withDescription("Output directory for compiled objects") .withLongOpt(BIN_OUT_DIR_ARG) .create()); + codeGenOpts.addOption(OptionBuilder + .withDescription("Delete compile directory") + .withLongOpt(DELETE_COMPILE_ARG) + .create()); codeGenOpts.addOption(OptionBuilder.withArgName("name") .hasArg() .withDescription("Put auto-generated classes in this package") @@ -1423,6 +1428,10 @@ protected void applyCodeGenOptions(CommandLine in, SqoopOptions out, out.setJarOutputDir(in.getOptionValue(BIN_OUT_DIR_ARG)); } + if (in.hasOption(DELETE_COMPILE_ARG)) { + out.setDeleteJarOutputDir(true); + } + if (in.hasOption(PACKAGE_NAME_ARG)) { out.setPackageName(in.getOptionValue(PACKAGE_NAME_ARG)); } diff --git a/src/java/org/apache/sqoop/util/DirCleanupHook.java b/src/java/org/apache/sqoop/util/DirCleanupHook.java new file mode 100644 index 00000000..120bf0d4 --- /dev/null +++ b/src/java/org/apache/sqoop/util/DirCleanupHook.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.util; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.File; +import java.io.IOException; + +/** + * This Hook is currently used to clean up the temp directory that is used + * to hold the generated JAR files for each table class under /tmp/sqoop-/compile. + * + * But it is generic so it can also be used to clean up other directories if needed + */ +public class DirCleanupHook extends Thread { + + private File dir; + + public static final Log LOG = LogFactory.getLog(DirCleanupHook.class.getName()); + + public DirCleanupHook(String dirPath) { + dir = new File(dirPath); + } + + public void run() { + try { + LOG.debug("Removing directory: " + dir + " in the clean up hook."); + FileUtils.deleteDirectory(dir); + } catch (IOException e) { + LOG.error("Unable to remove directory: " + dir + ". Error was: " + e.getMessage()); + } + } +} diff --git a/src/test/org/apache/sqoop/util/TestDirCleanupHook.java b/src/test/org/apache/sqoop/util/TestDirCleanupHook.java new file mode 100644 index 00000000..41146fd1 --- /dev/null +++ b/src/test/org/apache/sqoop/util/TestDirCleanupHook.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.util; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.junit.Assert.assertFalse; + +public class TestDirCleanupHook { + @Rule + public TemporaryFolder tmpFolder = new TemporaryFolder(); + + private DirCleanupHook dirCleanupHook; + + @Before + public void before() throws Exception { + // Make sure the directory is not empty. + tmpFolder.newFile(); + + dirCleanupHook = new DirCleanupHook(tmpFolder.getRoot().getAbsolutePath()); + } + + @Test + public void testDirCleanup() { + dirCleanupHook.run(); + + assertFalse(tmpFolder.getRoot().exists()); + } +}