SQOOP-75. Allow configuration of ManagerFactories through a configuration subdirectory.

Files in conf/managers.d/ are treated as configuration files that specify
classes for sqoop.connection.factories. If this property is unset, these
files are processed in order.
ClassLoaderStack no longer attempts to load a jar if the test class is
already available with the current set of ClassLoaders.

From: Aaron Kimball <aaron@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149966 13f79535-47bb-0310-9956-ffa450edef68

bin/configure-sqoop

@@ -113,6 +113,7 @@ fi
 add_to_classpath ${SQOOP_JAR_DIR}
 
 export SQOOP_CLASSPATH
+export SQOOP_CONF_DIR
 export SQOOP_JAR_DIR
 export SQOOP_SHIM_DIR
 export HADOOP_CLASSPATH="${SQOOP_CLASSPATH}:${HADOOP_CLASSPATH}"

build.xml

@@ -488,6 +488,7 @@
         <exclude name="tags" />
         <exclude name=".project" />
         <exclude name=".classpath" />
+        <exclude name="conf/managers.d/**" />
       </fileset>
     </copy>
 

conf/.gitignore

@@ -13,3 +13,4 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 /sqoop-site.xml
+/managers.d

conf/sqoop-site-template.xml

@@ -22,8 +22,16 @@
 <configuration>
 
   <!--
-    Override the value of this property to enable third-party ManagerFactory
-    plugins.
+    Set the value of this property to explicitly enable third-party
+    ManagerFactory plugins.
+
+    If this is not used, you can alternately specify a set of ManagerFactories
+    in the $SQOOP_CONF_DIR/managers.d/ subdirectory.  Each file should contain
+    one or more lines like:
+      manager.class.name[=/path/to/containing.jar]
+
+    Files will be consulted in lexicographical order only if this property
+    is unset.
   -->
   <!--
   <property>

src/java/com/cloudera/sqoop/ConnFactory.java

@@ -18,20 +18,30 @@
 
 package com.cloudera.sqoop;
 
+import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
 
 import com.cloudera.sqoop.manager.ConnManager;
 import com.cloudera.sqoop.manager.DefaultManagerFactory;
 import com.cloudera.sqoop.manager.ManagerFactory;
 import com.cloudera.sqoop.metastore.SessionData;
 
+import com.cloudera.sqoop.util.ClassLoaderStack;
+
 /**
  * Factory class to create the ConnManager type required
  * for the current import job.
@@ -70,6 +80,7 @@ public ConnFactory(Configuration conf) {
    * the factories list.
    */
   private void instantiateFactories(Configuration conf) {
+    loadManagersFromConfDir(conf);
     String [] classNameArray =
         conf.getStrings(FACTORY_CLASS_NAMES_KEY, DEFAULT_FACTORY_CLASS_NAMES);
 
@@ -108,5 +119,112 @@ public ConnManager getManager(SessionData data) throws IOException {
     throw new IOException("No manager for connect string: "
         + data.getSqoopOptions().getConnectString());
   }
+
+  /**
+   * Add a ManagerFactory class to the list that we instantiate.
+   * @param conf the Configuration to set.
+   * @param factory the ManagerFactory class name to add.
+   */
+  private void addManager(Configuration conf, String factory) {
+    String curVal = conf.get(FACTORY_CLASS_NAMES_KEY);
+    if (null == curVal) {
+      conf.set(FACTORY_CLASS_NAMES_KEY, factory);
+    } else {
+      conf.set(FACTORY_CLASS_NAMES_KEY, curVal + "," + factory);
+    }
+  }
+
+  /**
+   * Read the specified file and extract any ManagerFactory implementation
+   * names from there.
+   * @param conf the configuration to populate.
+   * @param f the file containing the configuration data to add.
+   */
+  private void addManagersFromFile(Configuration conf, File f) {
+    Reader r = null;
+    try {
+      // The file format is actually Java properties-file syntax.
+      r = new InputStreamReader(new FileInputStream(f));
+      Properties props = new Properties();
+      props.load(r);
+
+      for (Map.Entry<Object, Object> entry : props.entrySet()) {
+        // Each key is a ManagerFactory class name.
+        // Each value, if set, is the jar that contains it.
+        String factory = entry.getKey().toString();
+        addManager(conf, factory);
+
+        String jarName = entry.getValue().toString();
+        if (jarName.length() > 0) {
+          ClassLoaderStack.addJarFile(jarName, factory);
+          LOG.debug("Added factory " + factory + " in jar " + jarName
+              + " specified by " + f);
+        } else if (LOG.isDebugEnabled()) {
+          LOG.debug("Added factory " + factory + " specified by " + f);
+        }
+      }
+    } catch (IOException ioe) {
+      LOG.error("Error loading ManagerFactory information from file "
+          + f + ": " + StringUtils.stringifyException(ioe));
+    } finally {
+      if (null != r) {
+        try {
+          r.close();
+        } catch (IOException ioe) {
+          LOG.warn("Error closing file " + f + ": " + ioe);
+        }
+      }
+    }
+  }
+
+  /**
+   * If $SQOOP_CONF_DIR/managers.d/ exists and sqoop.connection.factories is
+   * not set, then we look through the files in that directory; they should
+   * contain lines of the form mgr.class.name[=/path/to/containing.jar].
+   *
+   * Put all mgr.class.names into the Configuration, and load any specified
+   * jars into the ClassLoader.
+   *
+   * @param conf the current configuration to populate with class names.
+   * @return conf again, after possibly populating sqoop.connection.factories.
+   */
+  private Configuration loadManagersFromConfDir(Configuration conf) {
+    if (conf.get(FACTORY_CLASS_NAMES_KEY) != null) {
+      LOG.debug(FACTORY_CLASS_NAMES_KEY + " is set; ignoring managers.d");
+      return conf;
+    }
+
+    String confDirName = System.getenv("SQOOP_CONF_DIR");
+    if (null == confDirName) {
+      LOG.warn("$SQOOP_CONF_DIR has not been set in the environment. "
+          + "Cannot check for additional configuration.");  
+      return conf;
+    }
+
+    File confDir = new File(confDirName);
+    File mgrDir = new File(confDir, "managers.d");
+
+    if (mgrDir.exists() && mgrDir.isDirectory()) {
+      // We have a managers.d subdirectory. Get the file list, sort it,
+      // and process them in order.
+      String [] fileNames = mgrDir.list();
+      Arrays.sort(fileNames);
+
+      for (String fileName : fileNames) {
+        File f = new File(mgrDir, fileName);
+        if (f.isFile()) {
+          addManagersFromFile(conf, f);
+        }
+      }
+
+      // Add the default MF.
+      addManager(conf, DEFAULT_FACTORY_CLASS_NAMES);
+    }
+
+    // Set the classloader in this configuration so that it will use
+    // the jars we just loaded in.
+    conf.setClassLoader(Thread.currentThread().getContextClassLoader());
+    return conf;
+  }
 }
 

src/java/com/cloudera/sqoop/util/ClassLoaderStack.java

@@ -59,9 +59,22 @@ public static void setCurrentClassLoader(ClassLoader cl) {
   public static ClassLoader addJarFile(String jarFile, String testClassName)
       throws IOException {
 
-    // load the classes from the ORM JAR file into the current VM.
     ClassLoader prevClassLoader =
         Thread.currentThread().getContextClassLoader();
+
+    if (null != testClassName) {
+      try {
+        // Test to see if testClassName is already available. If so, do not
+        // load this jar.
+        LOG.debug("Checking for existing class: " + testClassName);
+        Class.forName(testClassName, true, prevClassLoader);
+        LOG.debug("Class is already available. Skipping jar " + jarFile);
+        return prevClassLoader;
+      } catch (ClassNotFoundException cnfe) {
+        // Expected this; we need to load the jar. continue.
+      }
+    }
+
     String urlPath = "jar:file://" + new File(jarFile).getAbsolutePath() + "!/";
     LOG.debug("Attempting to load jar through URL: " + urlPath);
     LOG.debug("Previous classloader is " + prevClassLoader);