SQOOP-2342: Sqoop2: Provide an external application classloader

(Abraham Elmahrek via Jarek Jarcec Cecho)

common/src/main/java/org/apache/sqoop/error/code/CoreError.java

@@ -52,7 +52,10 @@ public enum CoreError implements ErrorCode {
   CORE_0007("System not initialized"),
 
   /** The system has not been reconfigured */
-  CORE_0008("System not reconfigured");
+  CORE_0008("System not reconfigured"),
+
+  /** Sqoop classpath error */
+  CORE_0009("Sqoop classpath error"),
 
   ;
 

core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java

@@ -27,7 +27,6 @@
 import java.util.ResourceBundle;
 import java.util.Set;
 
-import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.connector.spi.SqoopConnector;
@@ -162,11 +161,6 @@ public synchronized void initialize(boolean autoUpgrade) {
       LOG.trace("Begin connector manager initialization");
     }
 
-    // add external connectors into the class path
-    // NOTE: class loading happens later in the ConnectorHandler
-    ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(SqoopConfiguration.getInstance().getContext()
-        .getString(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, StringUtils.EMPTY));
-
     List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs();
 
     LOG.info("Connector config urls: " + connectorConfigs);

core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java

@@ -17,21 +17,16 @@
  */
 package org.apache.sqoop.connector;
 
-import org.apache.commons.lang.StringUtils;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.core.ConfigurationConstants;
 import org.apache.sqoop.error.code.ConnectorError;
 
 import java.io.File;
 import java.io.IOException;
-import java.net.MalformedURLException;
 import java.net.URL;
-import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Enumeration;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 
@@ -75,25 +70,6 @@ public static List<URL> getConnectorConfigs() {
     return connectorConfigs;
   }
 
-  public static Set<File> getConnectorJars(String path) {
-    if (StringUtils.isEmpty(path)) {
-      return null;
-    }
-    Set<File> jarFiles = new HashSet<File>();
-    File folder = new File(path);
-    if (folder.exists()) {
-      for (File file : folder.listFiles()) {
-        if (file.isDirectory()) {
-          jarFiles.addAll(getConnectorJars(file.getPath()));
-        }
-        if (file.getName().endsWith(".jar") && isConnectorJar(file)) {
-          jarFiles.add(file);
-        }
-      }
-    }
-    return jarFiles;
-  }
-
   static boolean isConnectorJar(File file) {
     try {
       @SuppressWarnings("resource")
@@ -103,44 +79,4 @@ static boolean isConnectorJar(File file) {
       throw new RuntimeException(e);
     }
   }
-
-  public static void addExternalConnectorsJarsToClasspath(String path) {
-    if (StringUtils.isEmpty(path)) {
-      return;
-    }
-
-    ClassLoader currentThreadClassLoader = Thread.currentThread().getContextClassLoader();
-    if (currentThreadClassLoader != null) {
-
-      // Add the 'org.apache.sqoop.connector.external.loadpath' to the classpath
-      // Chain the current thread classloader
-      ExternalConnectorJarFileLoader connectorUrlClassLoader = new ExternalConnectorJarFileLoader(new URL[] {},
-          currentThreadClassLoader);
-      // the property always holds a path to the folder containing the jars
-      Set<File> connectorJars = getConnectorJars(path);
-      if (connectorJars != null && !connectorJars.isEmpty()) {
-        for (File jar : connectorJars) {
-          connectorUrlClassLoader.addJarFile(jar.getPath());
-        }
-        // Replace the thread classloader- assuming there is permission to do so
-        Thread.currentThread().setContextClassLoader(connectorUrlClassLoader);
-      }
-    }
-  }
-
-  public static class ExternalConnectorJarFileLoader extends URLClassLoader {
-    public ExternalConnectorJarFileLoader(URL[] urls, ClassLoader parent) {
-      super(urls, parent);
-    }
-
-    public void addJarFile(String path) {
-      String urlPath = "jar:file://" + path + "!/";
-      try {
-        addURL(new URL(urlPath));
-      } catch (MalformedURLException e) {
-        throw new RuntimeException(e);
-      }
-    }
-
-  }
 }

core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java

@@ -77,11 +77,9 @@ public final class ConfigurationConstants {
       "org.apache.sqoop.driver.autoupgrade";
 
   /**
-   # Support loading external connector jars only
-   # The loader will look for sqoopconnector.properties file in the jar before loading
-   # "/path/to/external/connectors/": Add all the connector JARs in the specified folder
+   * Add external jars to application classpath.
    */
-  public static final String EXTERNAL_CONNECTOR_LOAD_PATH = "org.apache.sqoop.connector.external.loadpath";
+  public static final String CLASSPATH = "org.apache.sqoop.classpath.extra";
 
   /**
    * Enable Sqoop App to kill Tomcat in case that it will fail to load.

core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java

@@ -21,9 +21,18 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.apache.log4j.PropertyConfigurator;
 import org.apache.sqoop.classification.InterfaceAudience;
@@ -175,6 +184,8 @@ public synchronized void initialize() {
     provider.registerListener(new CoreConfigurationListener(SqoopConfiguration.getInstance()));
 
     initialized = true;
+
+    configureClassLoader(ConfigurationConstants.CLASSPATH);
   }
 
   public synchronized MapContext getContext() {
@@ -212,6 +223,48 @@ public synchronized void destroy() {
     initialized = false;
   }
 
+  /**
+   * Load extra classpath from sqoop configuration.
+   * @param classpathProperty
+   */
+  private synchronized void configureClassLoader(String classpathProperty) {
+    LOG.info("Adding jars to current classloader from property: " + classpathProperty);
+
+    String classpath = getContext().getString(classpathProperty);
+
+    if (StringUtils.isEmpty(classpath)) {
+      LOG.debug("Property " + classpathProperty + " is null or empty. Not adding any extra jars.");
+      return;
+    }
+
+    ClassLoader currentThreadClassLoader = Thread.currentThread().getContextClassLoader();
+    if (currentThreadClassLoader == null) {
+      throw new SqoopException(CoreError.CORE_0009, "No thread context classloader to override.");
+    }
+
+    // CSV URL list separated by ":".
+    Set<String> paths = new HashSet(Arrays.asList(classpath.split(":")));
+    List<URL> urls = new LinkedList<URL>();
+
+    for (String path : paths) {
+      try {
+        LOG.debug("Found jar in path: " + path);
+        URL url = new File(path).toURI().toURL();
+        urls.add(url);
+        LOG.debug("Using URL: " + url.toString());
+      } catch (MalformedURLException e) {
+        throw new SqoopException(CoreError.CORE_0009, "Malformed URL found.", e);
+      }
+    }
+
+    // Chain the current thread classloader so that
+    // configured classpath adds to existing classloader.
+    // Existing classpath is not changed.
+    URLClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]),
+        currentThreadClassLoader);
+    Thread.currentThread().setContextClassLoader(classLoader);
+  }
+
   private synchronized void configureLogging() {
     Properties props = new Properties();
     for (String key : config.keySet()) {

core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java

@@ -20,16 +20,10 @@
 
 import static org.testng.AssertJUnit.assertFalse;
 import static org.testng.AssertJUnit.assertTrue;
-import static org.testng.AssertJUnit.assertNull;
-import static org.testng.AssertJUnit.assertEquals;
-import org.apache.sqoop.utils.ClassUtils;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 import java.io.File;
-import java.net.URL;
-import java.util.List;
-import java.util.Set;
 
 public class TestConnectorManagerUtils {
 
@@ -40,19 +34,6 @@ public void setUp() {
     workingDir = System.getProperty("user.dir");
   }
 
-  @Test
-  public void testGetConnectorJarsNullPath() {
-    Set<File> files = ConnectorManagerUtils.getConnectorJars(null);
-    assertNull(files);
-  }
-
-  @Test
-  public void testGetConnectorJarsNonNullPath() {
-    String path = workingDir + "/src/test/resources";
-    Set<File> files = ConnectorManagerUtils.getConnectorJars(path);
-    assertEquals(1, files.size());
-  }
-
   @Test
   public void testIsConnectorJar() {
     String path = workingDir + "/src/test/resources/test-connector.jar";
@@ -69,13 +50,4 @@ public void testIsNotConnectorJar() {
     assertFalse(ConnectorManagerUtils.isConnectorJar(file));
   }
 
-  @Test
-  public void testAddExternalConnectorJarToClasspath() {
-    String path = workingDir + "/src/test/resources";
-    ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(path);
-    List<URL> urls = ConnectorManagerUtils.getConnectorConfigs();
-    assertEquals(1, urls.size());
-    ClassUtils.loadClass("org.apache.sqoop.connector.jdbc.GenericJdbcConnector");
-  }
-
 }

core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java

@@ -162,5 +162,8 @@ public void testSystemNotInitialized() throws Exception {
   public void testConfigurationInitSuccess() throws Exception {
     TestUtils.setupTestConfigurationWithExtraConfig(null, null);
     SqoopConfiguration.getInstance().initialize();
+
+    // Make sure "test" is in classpath.
+    Assert.assertNotNull(Thread.currentThread().getContextClassLoader().getResource("test_config.properties"));
   }
 }

core/src/test/resources/test_config.properties

@@ -23,3 +23,6 @@ org.apache.sqoop.log4j.logger.org.apache.sqoop=debug,console
 org.apache.sqoop.log4j.appender.console=org.apache.log4j.ConsoleAppender
 org.apache.sqoop.log4j.appender.console.layout=org.apache.log4j.PatternLayout
 org.apache.sqoop.log4j.appender.console.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+
+# Extra classpath
+org.apache.sqoop.classpath.extra=core/src/test/resources/test_config.properties

dist/src/main/server/conf/sqoop.properties

@@ -171,3 +171,8 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce
 # "/path/to/external/connectors/": Add all the connector JARs in the specified folder
 #
 org.apache.sqoop.connector.external.loadpath=
+
+# Sqoop application classpath
+# ":" separated list of jars to be included in sqoop.
+#
+org.apache.sqoop.classpath.extra=

docs/src/site/sphinx/ConnectorDevelopment.rst

@@ -483,15 +483,14 @@ Loading new connector say sqoop-foo-connector to the sqoop2, here are the steps
 
 1. Create a ``sqoop-foo-connector.jar``. Make sure the jar contains the ``sqoopconnector.properties`` for it to be picked up by Sqoop
 
-2. Add this jar to the a folder on your installation machine and update the path to this folder in the sqoop.properties located under the ``server/conf`` directory under the Sqoop2  for the key ``org.apache.sqoop.connector.external.loadpath``
+2. Add this jar to the ``org.apache.sqoop.classpath.extra`` property in the sqoop.properties located under the ``server/conf`` directory.
 
 ::
 
+ # Sqoop application classpath
+ # ":" separated list of jars to be included in sqoop.
  #
- # External connectors load path
- # "/path/to/external/connectors/": Add all the connector JARs in the specified folder
- #
- org.apache.sqoop.connector.external.loadpath=/path/to/connector
+ org.apache.sqoop.classpath.extra=/path/to/connector.jar
 
 3. Start the Sqoop 2 server and while initializing the server this jar should be loaded into the Sqoop 2's class path and registered into the Sqoop 2 repository
 

test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java

@@ -132,7 +132,6 @@ protected void prepareTemporaryPath() throws IOException {
     mapToProperties(sqoopProperties, getSecurityConfiguration());
     mapToProperties(sqoopProperties, getConnectorManagerConfiguration());
     mapToProperties(sqoopProperties, getDriverManagerConfiguration());
-    mapToProperties(sqoopProperties, getExternalConnectorLoadPathConfiguration());
 
     FileUtils.writeLines(f, sqoopProperties);
 
@@ -222,10 +221,4 @@ protected Map<String, String> getDriverManagerConfiguration() {
     properties.put(ConfigurationConstants.DRIVER_AUTO_UPGRADE, "true");
     return properties;
   }
-
-  protected Map<String, String> getExternalConnectorLoadPathConfiguration() {
-    Map<String, String> properties = new HashMap<String, String>();
-    properties.put(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, "");
-    return properties;
-  }
 }