SQOOP-3042: Sqoop does not clear compile directory under /tmp/sqoop-<username>/compile automatically

(Eric Lin via Szabolcs Vasas)

src/docs/man/codegen-args.txt

@@ -34,6 +34,9 @@ Code generation options
 --outdir (dir)::
   Output directory for generated code
 
+--delete-compile-dir::
+  Remove temporarily generated class Jar files after job finishes
+
 --package-name (package)::
   Puts auto-generated classes in the named Java package
 

src/docs/user/common-args.txt

@@ -34,6 +34,8 @@ Argument                                  Description
 +-P+                                      Read password from console
 +\--password <password>+                  Set authentication password
 +\--username <username>+                  Set authentication username
++\--delete-compile-dir+                   Remove temporarily generated class Jar\
+                                          files after job finishes
 +\--verbose+                              Print more information while working
 +\--connection-param-file <filename>+     Optional properties file that\
                                           provides connection parameters

src/java/org/apache/sqoop/SqoopOptions.java

@@ -204,6 +204,7 @@ public String toString() {
 
   @StoredAsProperty("codegen.output.dir") private String codeOutputDir;
   @StoredAsProperty("codegen.compile.dir") private String jarOutputDir;
+  @StoredAsProperty("codegen.delete.compile.dir") private boolean deleteJarOutputDir;
   // Boolean specifying whether jarOutputDir is a nonce tmpdir (true), or
   // explicitly set by the user (false). If the former, disregard any value
   // for jarOutputDir saved in the metastore.
@@ -1086,6 +1087,8 @@ private void initDefaults(Configuration baseConfiguration) {
     this.jarOutputDir = getNonceJarDir(tmpDir + "sqoop-" + localUsername
         + "/compile");
     this.jarDirIsAuto = true;
+    // Default to false, so behaviour does not change
+    this.deleteJarOutputDir = false;
     this.layout = FileLayout.TextFile;
 
     this.areOutputDelimsManuallySet = false;
@@ -1706,6 +1709,17 @@ public void setJarOutputDir(String outDir) {
     this.jarDirIsAuto = false;
   }
 
+  /**
+   * @return boolean - whether or not to delete the compile JAR directory
+   */
+  public Boolean getDeleteJarOutputDir() {
+    return this.deleteJarOutputDir;
+  }
+
+  public void setDeleteJarOutputDir(Boolean delete) {
+    this.deleteJarOutputDir = delete;
+  }
+
   /**
    * Return the value of $HADOOP_MAPRED_HOME.
    * @return $HADOOP_MAPRED_HOME, or null if it's not set.

src/java/org/apache/sqoop/orm/ClassWriter.java

@@ -51,6 +51,7 @@
 import org.apache.sqoop.lib.RecordParser;
 import org.apache.sqoop.lib.SqoopRecord;
 import org.apache.sqoop.manager.ConnManager;
+import org.apache.sqoop.util.DirCleanupHook;
 
 /**
  * Creates an ORM class to represent a table from a database.
@@ -1785,6 +1786,12 @@ public void generate() throws IOException {
       LOG.debug("sourceFilename is " + sourceFilename);
     }
 
+    // SQOOP-3042 - Sqoop does not clear compile directory under /tmp/sqoop-<username>/compile
+    // Add shutdown hook so that all files under the jar output dir can be cleared
+    if(options.getDeleteJarOutputDir()) {
+      Runtime.getRuntime().addShutdownHook(new DirCleanupHook(options.getJarOutputDir()));
+    }
+
     compileManager.addSourceFile(sourceFilename);
 
     // Create any missing parent directories.

src/java/org/apache/sqoop/orm/CompilationManager.java

@@ -245,13 +245,14 @@ public void compile() throws IOException {
         }
       }
       try {
-          FileUtils.moveFile(fOrig, fDest);
+        LOG.debug("moving file from " + fOrig.getAbsolutePath() + " to " + fDest.getAbsolutePath());
+        FileUtils.moveFile(fOrig, fDest);
       } catch (IOException e) {
-    	  /*Removed the exception being thrown
+        /*Removed the exception being thrown
-    	   *even if the .java file can not be renamed
+         *even if the .java file can not be renamed
-    	   *or can not be moved a "dest" directory for
+         *or can not be moved a "dest" directory for
-    	   *any reason.*/
+         *any reason.*/
-          LOG.debug("Could not rename " + orig + " to " + dest); 
+        LOG.error("Could not rename " + orig + " to " + dest + ". Error: " + e.getMessage());
       }
     }
   }

src/java/org/apache/sqoop/tool/BaseSqoopTool.java

@@ -102,6 +102,7 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool {
   public static final String TARGET_DIR_ARG = "target-dir";
   public static final String APPEND_ARG = "append";
   public static final String DELETE_ARG = "delete-target-dir";
+  public static final String DELETE_COMPILE_ARG = "delete-compile-dir";
   public static final String NULL_STRING = "null-string";
   public static final String INPUT_NULL_STRING = "input-null-string";
   public static final String NULL_NON_STRING = "null-non-string";
@@ -805,6 +806,10 @@ protected RelatedOptions getCodeGenOpts(boolean multiTable) {
         .withDescription("Output directory for compiled objects")
         .withLongOpt(BIN_OUT_DIR_ARG)
         .create());
+    codeGenOpts.addOption(OptionBuilder
+        .withDescription("Delete compile directory")
+        .withLongOpt(DELETE_COMPILE_ARG)
+        .create());
     codeGenOpts.addOption(OptionBuilder.withArgName("name")
         .hasArg()
         .withDescription("Put auto-generated classes in this package")
@@ -1423,6 +1428,10 @@ protected void applyCodeGenOptions(CommandLine in, SqoopOptions out,
       out.setJarOutputDir(in.getOptionValue(BIN_OUT_DIR_ARG));
     }
 
+    if (in.hasOption(DELETE_COMPILE_ARG)) {
+      out.setDeleteJarOutputDir(true);
+    }
+
     if (in.hasOption(PACKAGE_NAME_ARG)) {
       out.setPackageName(in.getOptionValue(PACKAGE_NAME_ARG));
     }

src/java/org/apache/sqoop/util/DirCleanupHook.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.util;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * This Hook is currently used to clean up the temp directory that is used
+ * to hold the generated JAR files for each table class under /tmp/sqoop-<username>/compile.
+ *
+ * But it is generic so it can also be used to clean up other directories if needed
+ */
+public class DirCleanupHook extends Thread {
+
+  private File dir;
+
+  public static final Log LOG = LogFactory.getLog(DirCleanupHook.class.getName());
+
+  public DirCleanupHook(String dirPath) {
+    dir = new File(dirPath);
+  }
+
+  public void run() {
+    try {
+      LOG.debug("Removing directory: " + dir + " in the clean up hook.");
+      FileUtils.deleteDirectory(dir);
+    } catch (IOException e) {
+      LOG.error("Unable to remove directory: " + dir + ". Error was: " + e.getMessage());
+    }
+  }
+}

src/test/org/apache/sqoop/util/TestDirCleanupHook.java

@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.util;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertFalse;
+
+public class TestDirCleanupHook {
+  @Rule
+  public TemporaryFolder tmpFolder = new TemporaryFolder();
+
+  private DirCleanupHook dirCleanupHook;
+
+  @Before
+  public void before() throws Exception {
+    // Make sure the directory is not empty.
+    tmpFolder.newFile();
+
+    dirCleanupHook = new DirCleanupHook(tmpFolder.getRoot().getAbsolutePath());
+  }
+
+  @Test
+  public void testDirCleanup() {
+    dirCleanupHook.run();
+
+    assertFalse(tmpFolder.getRoot().exists());
+  }
+}