SQOOP-1232. Sqoop2: Provide tooling infrastructure for Sqoop2

(Jarek Jarcec Cecho via Hari Shreedharan)

dist/pom.xml

@@ -166,6 +166,8 @@ limitations under the License.
                     </copy>
                     <copy file="../server/target/sqoop.war"
                       toDir="target/sqoop-${project.version}-bin-hadoop${hadoop.profile}/server/webapps"/>
+                    <copy file="../tomcat/target/sqoop-tomcat-${project.version}.jar"
+                      toDir="target/sqoop-${project.version}-bin-hadoop${hadoop.profile}/server/lib"/>
 
                     <!-- Build shell client directory -->
                     <copy todir="target/sqoop-${project.version}-bin-hadoop${hadoop.profile}/shell/lib">

dist/src/main/bin/sqoop.sh

@@ -68,6 +68,20 @@ setup_catalina_opts() {
 
 COMMAND=$1
 case $COMMAND in
+  tool)
+    if [ $# = 1 ]; then
+      echo "Usage: sqoop.sh tool TOOL_NAME [TOOL_ARGS]"
+      exit
+    fi
+
+    source ${BASEDIR}/bin/sqoop-sys.sh
+    setup_catalina_opts
+
+    # Remove the "tool" keyword from the command line and pass the rest
+    shift
+
+    $CATALINA_BIN/tool-wrapper.sh -server org.apache.sqoop.tomcat.TomcatToolRunner $@
+    ;;
   server)
     if [ $# = 1 ]; then
       echo "Usage: sqoop.sh server <start/stop>"

pom.xml

@@ -14,10 +14,7 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+--><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache</groupId>
@@ -310,6 +307,11 @@ limitations under the License.
         <artifactId>sqoop-connector-mysql-jdbc</artifactId>
         <version>${project.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.sqoop</groupId>
+        <artifactId>sqoop-tools</artifactId>
+        <version>${project.version}</version>
+      </dependency>
       <dependency>
         <groupId>com.googlecode.json-simple</groupId>
         <artifactId>json-simple</artifactId>
@@ -413,6 +415,12 @@ limitations under the License.
       <version>${mockito.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.tomcat</groupId>
+      <artifactId>catalina</artifactId>
+      <version>${tomcat.version}</version>
+      <scope>provided</scope>
+    </dependency>
    </dependencies>
   </dependencyManagement>
 
@@ -423,13 +431,15 @@ limitations under the License.
     <module>repository</module>
     <module>server</module>
     <module>client</module>
-	<module>shell</module>
+    <module>shell</module>
     <module>docs</module>
     <module>connector</module>
     <module>execution</module>
     <module>submission</module>
     <module>dist</module>
     <module>test</module>
+    <module>tools</module>
+    <module>tomcat</module>
   </modules>
 
   <build>
@@ -519,7 +529,7 @@ limitations under the License.
         <version>2.4</version>
         <reportSets>
           <reportSet>
-            <reports></reports>
+            <reports/>
           </reportSet>
         </reportSets>
       </plugin>

server/pom.xml

@@ -38,6 +38,11 @@ limitations under the License.
       <artifactId>sqoop-core</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.sqoop</groupId>
+      <artifactId>sqoop-tools</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.sqoop</groupId>
       <artifactId>sqoop-common</artifactId>

tomcat/pom.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache</groupId>
+    <artifactId>sqoop</artifactId>
+    <version>2.0.0-SNAPSHOT</version>
+  </parent>
+
+  <groupId>org.apache.sqoop</groupId>
+  <artifactId>sqoop-tomcat</artifactId>
+  <name>Sqoop Tomcat additions</name>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.tomcat</groupId>
+      <artifactId>catalina</artifactId>
+    </dependency>
+
+  </dependencies>
+</project>

tomcat/src/main/java/org/apache/sqoop/tomcat/TomcatToolRunner.java

@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.tomcat;
+
+import org.apache.catalina.Host;
+import org.apache.catalina.startup.Bootstrap;
+import org.apache.catalina.startup.ClassLoaderFactory;
+import org.apache.catalina.startup.Embedded;
+import org.apache.catalina.startup.ExpandWar;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.net.URL;
+
+/**
+ * Add Sqoop webapp and common loader into the classpath and run the usual ToolRunner.
+ *
+ * This class will be executed via Tomcat Tool mechanism that do a lot of heavy
+ * lifting for us - it will set up most of the environment and class loaders. Sadly
+ * it won't setup the common.loader (Hadoop dependencies) and the Sqoop webapp itself.
+ */
+public class TomcatToolRunner {
+
+  // TODO: The appBase can be loaded from the conf/server.xml file
+  private static String PROPERTY_APPBASE_PATH = "org.apache.sqoop.tomcat.webapp.path";
+  private static String DEFAULT_APPBASE_PATH = "webapps";
+
+  public static void main(String[] args) throws Exception {
+    // Using Boostrap class to boot the common.loader and other catalina specific
+    // class loaders.
+    Bootstrap bootstrap = new Bootstrap();
+    bootstrap.init();
+
+    // Now we need to add the sqoop webapp classes into the class loader. Sadly
+    // we have to do a lot of things ourselves. The procedure is:
+    // 1) Unpack Sqoop war file
+    // 2) Build the ClassLoader using Tomcat's ClassLoaderFactory
+
+    // Various paths to war file and locations inside the war file
+    String webappPath = System.getProperty(PROPERTY_APPBASE_PATH, DEFAULT_APPBASE_PATH);
+    String catalinaBase = Bootstrap.getCatalinaHome();
+    String fullWebappPath = catalinaBase + File.separator + webappPath;
+    String fullSqoopWarPath = fullWebappPath + File.separator + "sqoop.war";
+    String fullSqoopClassesPath = fullWebappPath + File.separator + "sqoop" + File.separator + "WEB-INF" + File.separator + "classes";
+    String fullSqoopLibPath = fullWebappPath + File.separator + "sqoop" + File.separator + "WEB-INF" + File.separator + "lib";
+
+    // Expand the war into the usual location, this operation is idempotent (nothing bad happens if it's already expanded)
+    Embedded embedded = new Embedded();
+    Host host = embedded.createHost("Sqoop Tool Virtual Host", fullWebappPath);
+    ExpandWar.expand(host, new URL("jar:file://" + fullSqoopWarPath + "!/"));
+
+    // We have expanded war file, so we build the classloader from
+    File [] unpacked = new File[1];   unpacked[0] = new File(fullSqoopClassesPath);
+    File [] packed = new File[1];     packed[0] = new File(fullSqoopLibPath);
+    ClassLoader loader = ClassLoaderFactory.createClassLoader(unpacked, packed, Thread.currentThread().getContextClassLoader());
+    Thread.currentThread().setContextClassLoader(loader);
+
+    // Finally we can call the usual ToolRunner. We have to use reflection as
+    // as the time of loading this class, Sqoop dependencies are not on classpath.
+    Class klass = Class.forName("org.apache.sqoop.tools.ToolRunner", true, loader);
+    Method method = klass.getMethod("main", String[].class);
+    method.invoke(null, (Object)args);
+  }
+
+}

tools/pom.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache</groupId>
+    <artifactId>sqoop</artifactId>
+    <version>2.0.0-SNAPSHOT</version>
+  </parent>
+
+  <groupId>org.apache.sqoop</groupId>
+  <artifactId>sqoop-tools</artifactId>
+  <name>Sqoop Tools</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.sqoop</groupId>
+      <artifactId>sqoop-core</artifactId>
+    </dependency>
+
+  </dependencies>
+
+</project>

tools/src/main/java/org/apache/sqoop/tools/Tool.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.tools;
+
+/**
+ * Abstract class describing Sqoop tool.
+ *
+ * Tools are server side actions that administrator might need to execute as
+ * one time action. They do not serve as a service and are not participating
+ * in data transfers (unlike Sqoop 1 tools). They are strictly maintenance
+ * related such as "validate deployment" or "upgrade repository".
+ *
+ * All tools are executed in the same environment as Sqoop server itself (classpath,
+ * configuration, java properties, ...).
+ */
+public abstract class Tool {
+
+  /**
+   * Run the tool.
+   *
+   * @param arguments Arguments as were entered on the command line
+   * @return true if the execution was successful
+   */
+  abstract public boolean runTool(String[] arguments);
+}

tools/src/main/java/org/apache/sqoop/tools/ToolRunner.java

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.tools;
+
+import org.apache.sqoop.common.VersionInfo;
+import org.apache.sqoop.tools.tool.BuiltinTools;
+import org.apache.sqoop.utils.ClassUtils;
+
+import java.util.Arrays;
+
+/**
+ * Runner allowing the tool to be executed from command line.
+ *
+ * It's expected that all usual dependencies are already on the classpath.
+ */
+public final class ToolRunner {
+
+  /**
+   * Run tool given as first argument and pass all other arguments to it as
+   * tool parameters.
+   *
+   * @param args Tool name and it's arguments
+   * @throws Exception
+   */
+  public static void main(String[] args) throws Exception {
+    System.out.println("Sqoop tool executor:");
+    System.out.println("\tVersion: " + VersionInfo.getVersion());
+    System.out.println("\tRevision: " + VersionInfo.getRevision());
+    System.out.println("\tCompiled on " + VersionInfo.getDate() + " by " + VersionInfo.getUser());
+
+    if(args.length < 1) {
+      throw new IllegalArgumentException("Name of the tool is missing.");
+    }
+
+    Class<? extends Tool> toolClass;
+
+    // Firstly let's try to resolve short name
+    toolClass = BuiltinTools.getTool(args[0]);
+
+    // If the short name wasn't resolved, find the class by it's name
+    if(toolClass == null) {
+      toolClass = (Class<? extends Tool>) ClassUtils.loadClass(args[0]);
+    }
+
+    if(toolClass == null) {
+      throw new IllegalArgumentException("Can't find tool: " + args[0]);
+    }
+
+    Tool tool = (Tool) ClassUtils.instantiate(toolClass);
+    if(tool == null) {
+      throw new RuntimeException("Can't get tool instance: " + args[0]);
+    }
+
+    System.out.println("Running tool: " + toolClass);
+    if(tool.runTool(Arrays.copyOfRange(args, 1, args.length))) {
+      System.out.println("Tool " + toolClass + " has finished correctly.");
+    } else {
+      System.out.println("Tool " + toolClass + " has failed.");
+    }
+  }
+
+  private ToolRunner() {
+    // Instantiation is prohibited
+  }
+}

tools/src/main/java/org/apache/sqoop/tools/tool/BuiltinTools.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.tools.tool;
+
+import org.apache.sqoop.tools.Tool;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * List of built-in tools with short names for easier manipulation.
+ */
+public class BuiltinTools {
+
+  /**
+   * List of built-in tools
+   */
+  private static Map<String, Class<? extends Tool>> tools;
+  static {
+    tools = new HashMap<String, Class<? extends Tool>>();
+    tools.put("verify", VerifyTool.class);
+  }
+
+  /**
+   * Returns the tool class associated with the short name or NULL in case that
+   * the short name is not known.
+   *
+   * @param name Short name for the built-in tool
+   * @return
+   */
+  public static Class<? extends Tool> getTool(String name) {
+    return tools.get(name);
+  }
+
+  private BuiltinTools() {
+    // Instantiation is forbidden
+  }
+}

tools/src/main/java/org/apache/sqoop/tools/tool/VerifyTool.java

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.tools.tool;
+
+import org.apache.sqoop.core.SqoopServer;
+import org.apache.sqoop.tools.Tool;
+import org.apache.log4j.Logger;
+
+/**
+ * Try to initialize all Sqoop sub systems to verify that Sqoop 2 is correctly
+ * configured and then tear it down. This tool will start and stop all subsystems
+ * with the exception of servlets.
+ */
+public class VerifyTool extends Tool {
+
+  public static final Logger LOG = Logger.getLogger(VerifyTool.class);
+
+  @Override
+  public boolean runTool(String[] arguments) {
+    try {
+      SqoopServer.initialize();
+      SqoopServer.destroy();
+      System.out.println("Verification was successful.");
+      return true;
+    } catch(Exception ex) {
+      LOG.error("Got exception while initializing/destroying Sqoop server:", ex);
+      System.out.println("Verification has failed, please check Server logs for further details.");
+      return false;
+    }
+  }
+
+}