SQOOP-45. If compilation dir is unspecified, use a nonce directory.

.class and .jar generation occurs in a per-process anonymous
directory. Generation of .java files now occurs in the bin directory;
.java files are moved to the src directory after compilation.

From: Aaron Kimball <aaron@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149947 13f79535-47bb-0310-9956-ffa450edef68

src/java/com/cloudera/sqoop/SqoopOptions.java

@@ -31,6 +31,8 @@
 import com.cloudera.sqoop.lib.DelimiterSet;
 import com.cloudera.sqoop.lib.LargeObjectLoader;
 
+import com.cloudera.sqoop.util.RandomHash;
+
 /**
  * Configurable state used by Sqoop tools.
  */
@@ -186,6 +188,11 @@ public enum IncrementalMode {
   // 'parent' SqoopOptions instance, here.
   private SqoopOptions parent;
 
+  // Nonce directory name. Generate one per process, lazily, if
+  // getNonceJarDir() is called. Not recorded in metadata. This is used as
+  // a temporary holding area for compilation work done by this process.
+  private static String curNonce;
+
   public SqoopOptions() {
     initDefaults(null);
   }
@@ -581,6 +588,56 @@ public String getTempDir() {
     return this.tmpDir;
   }
 
+  /**
+   * Return the name of a directory that does not exist before 
+   * calling this method, and does exist afterward. We should be
+   * the only client of this directory.
+   */
+  private static String getNonceJarDir(String tmpBase) {
+
+    // Make sure we don't loop forever in the event of a permission error.
+    final int MAX_DIR_CREATE_ATTEMPTS = 32;
+
+    if (null != curNonce) {
+      return curNonce;
+    }
+   
+    File baseDir = new File(tmpBase);
+    File hashDir = null;
+
+    for (int attempts = 0; attempts < MAX_DIR_CREATE_ATTEMPTS; attempts++) {
+      hashDir = new File(baseDir, RandomHash.generateMD5String());
+      while (hashDir.exists()) {
+        hashDir = new File(baseDir, RandomHash.generateMD5String());
+      }
+
+      if (hashDir.mkdirs()) {
+        // We created the directory. Use it.
+        break;
+      }
+    }
+
+    if (hashDir == null || !hashDir.exists()) {
+      throw new RuntimeException("Could not create temporary directory: "
+          + hashDir + "; check for a directory permissions issue on /tmp.");
+    }
+
+    LOG.debug("Generated nonce dir: " + hashDir.toString());
+    SqoopOptions.curNonce = hashDir.toString();
+    return SqoopOptions.curNonce;
+  }
+
+  /**
+   * Reset the nonce directory and force a new one to be generated. This
+   * method is intended to be used only by multiple unit tests that want
+   * to isolate themselves from one another. It should not be called
+   * during normal Sqoop execution.
+   */
+  public static void clearNonceDir() {
+    LOG.warn("Clearing nonce directory");
+    SqoopOptions.curNonce = null;
+  }
+
   private void initDefaults(Configuration baseConfiguration) {
     // first, set the true defaults if nothing else happens.
     // default action is to run the full pipeline.
@@ -604,7 +661,7 @@ private void initDefaults(Configuration baseConfiguration) {
     }
 
     this.tmpDir = myTmpDir;
-    this.jarOutputDir = tmpDir + "sqoop/compile";
+    this.jarOutputDir = getNonceJarDir(tmpDir + "sqoop/compile");
     this.layout = FileLayout.TextFile;
 
     this.areDelimsManuallySet = false;

src/java/com/cloudera/sqoop/io/LobFile.java

@@ -31,9 +31,6 @@
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.UnsupportedEncodingException;
-import java.rmi.server.UID;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
 import java.util.AbstractMap;
 import java.util.Arrays;
 import java.util.ArrayList;
@@ -66,6 +63,8 @@
 import org.apache.hadoop.io.compress.Decompressor;
 import org.apache.hadoop.io.compress.DecompressorStream;
 
+import com.cloudera.sqoop.util.RandomHash;
+
 /**
  * File format which stores large object records.
  * The format allows large objects to be read through individual InputStreams
@@ -236,14 +235,7 @@ public void write(DataOutput out) throws IOException {
      * Generate a new random RecordStartMark.
      */
     private void generateStartMark() {
-      try {
+      this.startBytes = RandomHash.generateMD5Bytes();
-        MessageDigest digester = MessageDigest.getInstance("MD5");
-        long time = System.currentTimeMillis();
-        digester.update((new UID() + "@" + time).getBytes());
-        this.startBytes = digester.digest();
-      } catch (NoSuchAlgorithmException e) {
-        throw new RuntimeException(e);
-      }
     }
   }
 

src/java/com/cloudera/sqoop/orm/ClassWriter.java

@@ -972,8 +972,9 @@ public void generate() throws IOException {
     StringBuilder sb = generateClassForColumns(columnTypes,
         cleanedColNames, cleanedDbWriteColNames);
 
-    // Write this out to a file.
+    // Write this out to a file in the jar output directory.
-    String codeOutDir = options.getCodeOutputDir();
+    // We'll move it to the user-visible CodeOutputDir after compiling.
+    String codeOutDir = options.getJarOutputDir();
 
     // Get the class name to generate, which includes package components.
     String className = new TableClassName(options).getClassForTable(tableName);
@@ -999,11 +1000,11 @@ public void generate() throws IOException {
 
     // Create any missing parent directories.
     File file = new File(filename);
-    String dirname = file.getParent();
+    File dir = file.getParentFile();
-    if (null != dirname) {
+    if (null != dir && !dir.exists()) {
-      boolean mkdirSuccess = new File(dirname).mkdirs();
+      boolean mkdirSuccess = dir.mkdirs();
       if (!mkdirSuccess) {
-        LOG.debug("Could not create directory tree for " + dirname);
+        LOG.debug("Could not create directory tree for " + dir);
       }
     }
 

src/java/com/cloudera/sqoop/orm/CompilationManager.java

@@ -109,6 +109,7 @@ private String findHadoopCoreJar() {
 
   /**
    * Compile the .java files into .class files via embedded javac call.
+   * On success, move .java files to the code output dir.
    */
   public void compile() throws IOException {
     List<String> args = new ArrayList<String>();
@@ -125,6 +126,11 @@ public void compile() throws IOException {
       LOG.debug("Found existing " + jarOutDir);
     }
 
+    // Make sure jarOutDir ends with a '/'.
+    if (!jarOutDir.endsWith(File.separator)) {
+      jarOutDir = jarOutDir + File.separator;
+    }
+
     // find hadoop-*-core.jar for classpath.
     String coreJar = findHadoopCoreJar();
     if (null == coreJar) {
@@ -152,13 +158,8 @@ public void compile() throws IOException {
 
     String curClasspath = System.getProperty("java.class.path");
 
-    String srcOutDir = new File(options.getCodeOutputDir()).getAbsolutePath();
-    if (!srcOutDir.endsWith(File.separator)) {
-      srcOutDir = srcOutDir + File.separator;
-    }
-
     args.add("-sourcepath");
-    args.add(srcOutDir);
+    args.add(jarOutDir);
 
     args.add("-d");
     args.add(jarOutDir);
@@ -178,8 +179,8 @@ public void compile() throws IOException {
 
     ArrayList<String> srcFileNames = new ArrayList<String>();
     for (String srcfile : sources) {
-      srcFileNames.add(srcOutDir + srcfile);
+      srcFileNames.add(jarOutDir + srcfile);
-      LOG.debug("Adding source file: " + srcOutDir + srcfile);
+      LOG.debug("Adding source file: " + jarOutDir + srcfile);
     }
 
     if (LOG.isDebugEnabled()) {
@@ -203,6 +204,30 @@ public void compile() throws IOException {
     if (!result) {
       throw new IOException("Error returned by javac");
     }
+
+    // Where we should move source files after compilation.
+    String srcOutDir = new File(options.getCodeOutputDir()).getAbsolutePath();
+    if (!srcOutDir.endsWith(File.separator)) {
+      srcOutDir = srcOutDir + File.separator;
+    }
+
+    // Move these files to the srcOutDir.
+    for (String srcFileName : sources) {
+      String orig = jarOutDir + srcFileName;
+      String dest = srcOutDir + srcFileName;
+      File fOrig = new File(orig);
+      File fDest = new File(dest);
+      File fDestParent = fDest.getParentFile();
+      if (null != fDestParent && !fDestParent.exists()) {
+        if (!fDestParent.mkdirs()) {
+          LOG.error("Could not make directory: " + fDestParent);
+        }
+      }
+
+      if (!fOrig.renameTo(fDest)) {
+        LOG.error("Could not rename " + orig + " to " + dest);
+      }
+    }
   }
 
   /**

src/java/com/cloudera/sqoop/util/RandomHash.java

@@ -0,0 +1,62 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.sqoop.util;
+
+import java.rmi.server.UID;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * Securely generate random MD5 signatures for use as nonce values.
+ */
+public final class RandomHash {
+
+  private RandomHash() {
+  }
+
+  /**
+   * Generate a new random md5 hash.
+   * @return a securely-generated random 16 byte sequence.
+   */
+  public static byte [] generateMD5Bytes() {
+    try {
+      MessageDigest digester = MessageDigest.getInstance("MD5");
+      long time = System.currentTimeMillis();
+      digester.update((new UID() + "@" + time).getBytes());
+      return digester.digest();
+    } catch (NoSuchAlgorithmException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Generate a new random md5 hash and convert it to a string.
+   * @return a securely-generated random string.
+   */
+  public static String generateMD5String() {
+    byte [] bytes = generateMD5Bytes();
+    StringBuilder sb = new StringBuilder();
+    for (byte b : bytes) {
+      int x = ((int) b) & 0xFF;
+      sb.append(String.format("%02x", x));
+    }
+    return sb.toString();
+  }
+}
+