SQOOP-910: Sqoop2: Move to embedded jetty from tomcat

(Colin Ma via Jarek Jarcec Cecho)

core/src/main/java/org/apache/sqoop/audit/AuditLoggerManager.java

@@ -64,11 +64,13 @@ public static AuditLoggerManager getInstance() {
   }
 
   private AuditLoggerManager() {
-    loggers = new ArrayList<AuditLogger>();
   }
 
   public synchronized void initialize() {
     LOG.info("Begin audit logger manager initialization");
+    if (loggers == null) {
+      loggers = new ArrayList<AuditLogger>();
+    }
     initializeLoggers();
 
     SqoopConfiguration.getInstance().getProvider()

core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java

@@ -94,11 +94,11 @@ public static void setInstance(ConnectorManager newInstance) {
   }
 
   // key: connector id, value: connector name
-  private Map<Long, String> idToNameMap = new HashMap<Long, String>();
+  private Map<Long, String> idToNameMap;
   private Set<String> connectorNames = new HashSet<String>();
 
   // key: connector name, value: connector handler
-  private Map<String, ConnectorHandler> handlerMap = new HashMap<String, ConnectorHandler>();
+  private Map<String, ConnectorHandler> handlerMap;
 
   public List<MConnector> getConnectorConfigurables() {
     List<MConnector> connectors = new LinkedList<MConnector>();
@@ -112,10 +112,6 @@ public Set<Long> getConnectorIds() {
     return idToNameMap.keySet();
   }
 
-  public Set<String> getConnectorNames() {
-    return connectorNames;
-  }
-
   public Map<Long, ResourceBundle> getResourceBundles(Locale locale) {
     Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
     for (ConnectorHandler handler : handlerMap.values()) {
@@ -166,6 +162,16 @@ public synchronized void initialize() {
   }
 
   public synchronized void initialize(boolean autoUpgrade) {
+    if (handlerMap == null) {
+      handlerMap = new HashMap<String, ConnectorHandler>();
+    }
+    if (idToNameMap == null) {
+      idToNameMap = new HashMap<Long, String>();
+    }
+    if (connectorNames == null) {
+      connectorNames = new HashSet<String>();
+    }
+
     if (LOG.isTraceEnabled()) {
       LOG.trace("Begin connector manager initialization");
     }
@@ -234,8 +240,9 @@ private synchronized void registerConnectors(boolean autoUpgrade) {
   }
 
   public synchronized void destroy() {
-      handlerMap = null;
-      idToNameMap = null;
+    handlerMap = null;
+    idToNameMap = null;
+    connectorNames = null;
   }
 
   @Override

core/src/main/java/org/apache/sqoop/core/SqoopServer.java

@@ -49,7 +49,7 @@ public static void destroy() {
 
   public static void initialize() {
     try {
-      LOG.info("Booting up Sqoop server");
+      LOG.info("Initializing Sqoop server.");
       SqoopConfiguration.getInstance().initialize();
       AuthenticationManager.getInstance().initialize();
       AuthorizationManager.getInstance().initialize();
@@ -58,9 +58,9 @@ public static void initialize() {
       ConnectorManager.getInstance().initialize();
       Driver.getInstance().initialize();
       JobManager.getInstance().initialize();
-      LOG.info("Sqoop server has successfully boot up");
+      LOG.info("Sqoop server has successfully been initialized.");
     } catch (RuntimeException | ClassNotFoundException | IllegalAccessException | InstantiationException e) {
-      LOG.error("Server startup failure", e);
+      LOG.error("Failure in server initialization", e);
       throw new RuntimeException("Failure in server initialization", e);
     }
   }

core/src/main/java/org/apache/sqoop/driver/JobManager.java

@@ -138,7 +138,7 @@ public static void setInstance(JobManager newInstance) {
   /**
    * Synchronization variable between threads.
    */
-  private boolean running = true;
+  private boolean running;
 
   /**
    * Specifies how old submissions should be removed from repository.
@@ -215,7 +215,7 @@ public synchronized void destroy() {
   public synchronized void initialize() {
     LOG.trace("Begin submission engine manager initialization");
     MapContext context = SqoopConfiguration.getInstance().getContext();
-
+    running = true;
     // Let's load configured submission engine
     String submissionEngineClassName =
       context.getString(DriverConstants.SYSCFG_SUBMISSION_ENGINE);

dist/pom.xml

@@ -40,7 +40,6 @@ limitations under the License.
     <dependency>
       <groupId>org.apache.sqoop</groupId>
       <artifactId>sqoop-server</artifactId>
-      <type>war</type>
     </dependency>
     <dependency>
       <groupId>org.apache.sqoop</groupId>
@@ -155,27 +154,13 @@ limitations under the License.
                       </fileset>
                      </copy>
 
-                    <!-- Build server directory -->
-                    <get src="http://archive.apache.org/dist/tomcat/tomcat-${tomcat.major.version}/v${tomcat.version}/bin/apache-tomcat-${tomcat.version}.tar.gz"
-                         dest="target/apache-tomcat.tar.gz" verbose="true" skipexisting="true"/>
-                    <untar src="target/apache-tomcat.tar.gz" dest="target" compression="gzip"/>
-                    <move file="target/apache-tomcat-${tomcat.version}" tofile="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server"/>
-                    <copy todir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server/bin">
-                      <fileset dir="src/main/server/bin"/>
+                    <copy todir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server/lib">
+                      <fileset dir="../server/target/lib">
+                        <include name="*.jar" />
+                      </fileset>
                     </copy>
-                    <chmod dir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server/bin" perm="u+x" includes="**/*.sh"/>
-                    <copy todir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server/conf">
-                      <fileset dir="src/main/server/conf"/>
-                    </copy>
-                    <delete dir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server/webapps"/>
-                    <mkdir dir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server/webapps"/>
-                    <copy todir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server/webapps">
-                      <fileset dir="src/main/server/webapps"/>
-                    </copy>
-                    <copy file="../server/target/sqoop.war"
-                      toDir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server/webapps"/>
-                    <copy file="../tomcat/target/sqoop-tomcat-${project.version}.jar"
-                      toDir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server/lib"/>
+                    <copy file="../server/target/sqoop-server-${project.version}.jar"
+                            toDir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/server/lib"/>
 
                     <!-- Build shell client directory -->
                     <copy todir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/shell/lib">
@@ -188,6 +173,20 @@ limitations under the License.
                     <copy file="../shell/target/sqoop-shell-${project.version}.jar"
                       toDir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/shell/lib"/>
 
+                    <!-- Build tools client directory -->
+                    <copy todir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/tools/lib">
+                      <fileset dir="../tools/target/lib">
+                        <include name="*.jar" />
+                        </fileset>
+                    </copy>
+                    <copy file="../tools/target/sqoop-tools-${project.version}.jar"
+                          toDir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/tools/lib"/>
+
+                    <!-- Build "conf" directory -->
+                    <copy todir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/conf">
+                        <fileset dir="src/main/conf"/>
+                    </copy>
+
                     <!-- Build "bin" directory -->
                     <copy todir="target/sqoop-${project.version}-bin-hadoop${hadoop.version}/bin">
                       <fileset dir="src/main/bin"/>

dist/src/main/bin/sqoop-sys.sh

@@ -27,17 +27,3 @@ ECHO="true"
 if [ "${1}" == "-silent" ]; then
   ECHO="false"
 fi
-
-if [ "${SQOOP_HTTP_PORT}" = "" ]; then
-  export SQOOP_HTTP_PORT=12000
-  print "Setting SQOOP_HTTP_PORT:     ${SQOOP_HTTP_PORT}"
-else
-  print "Using   SQOOP_HTTP_PORT:     ${SQOOP_HTTP_PORT}"
-fi
-
-if [ "${SQOOP_ADMIN_PORT}" = "" ]; then
-  export SQOOP_ADMIN_PORT=`expr $SQOOP_HTTP_PORT +  1`
-  print "Setting SQOOP_ADMIN_PORT:     ${SQOOP_ADMIN_PORT}"
-else
-  print "Using   SQOOP_ADMIN_PORT:     ${SQOOP_ADMIN_PORT}"
-fi

dist/src/main/bin/sqoop.sh

@@ -24,6 +24,80 @@ function print_usage(){
   echo ""
 }
 
+function sqoop_server_classpath_set {
+
+  HADOOP_COMMON_HOME=${HADOOP_COMMON_HOME:-${HADOOP_HOME}/share/hadoop/common}
+  HADOOP_HDFS_HOME=${HADOOP_HDFS_HOME:-${HADOOP_HOME}/share/hadoop/hdfs}
+  HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-${HADOOP_HOME}/share/hadoop/mapreduce}
+  HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-${HADOOP_HOME}/share/hadoop/yarn}
+
+  if [[ ! (-d "${HADOOP_COMMON_HOME}" && -d "${HADOOP_HDFS_HOME}" && -d "${HADOOP_MAPRED_HOME}" && -d "${HADOOP_YARN_HOME}") ]]; then
+    echo "Can't load the Hadoop related java lib, please check the setting for the following environment variables:"
+    echo "    HADOOP_COMMON_HOME, HADOOP_HDFS_HOME, HADOOP_MAPRED_HOME, HADOOP_YARN_HOME"
+    exit
+  fi
+
+  for f in $SQOOP_SERVER_LIB/*.jar; do
+    CLASSPATH="${CLASSPATH}:$f"
+  done
+
+  for f in $HADOOP_COMMON_HOME/*.jar; do
+    CLASSPATH="${CLASSPATH}:$f"
+  done
+
+  for f in $HADOOP_COMMON_HOME/lib/*.jar; do
+    CLASSPATH="${CLASSPATH}:$f"
+  done
+
+  for f in $HADOOP_HDFS_HOME/*.jar; do
+    CLASSPATH="${CLASSPATH}:$f"
+  done
+
+  for f in $HADOOP_HDFS_HOME/lib/*.jar; do
+    CLASSPATH="${CLASSPATH}:$f"
+  done
+
+  for f in $HADOOP_MAPRED_HOME/*.jar; do
+    CLASSPATH="${CLASSPATH}:$f"
+  done
+
+  for f in $HADOOP_MAPRED_HOME/lib/*.jar; do
+    CLASSPATH="${CLASSPATH}:$f"
+  done
+
+  for f in $HADOOP_YARN_HOME/*.jar; do
+    CLASSPATH="${CLASSPATH}:$f"
+  done
+
+  for f in $HADOOP_YARN_HOME/lib/*.jar; do
+    CLASSPATH="${CLASSPATH}:$f"
+  done
+
+  for f in $HIVE_HOME/lib/*.jar; do
+    # exclude the jdbc for derby, to avoid the sealing violation exception
+    if [[ ! $f =~ derby* && ! $f =~ jetty* ]]; then
+      CLASSPATH="${CLASSPATH}:$f"
+    fi
+  done
+}
+
+function is_sqoop_server_running {
+  if [[ -f "${sqoop_pidfile}" ]]; then
+    kill -s 0 $(cat "$sqoop_pidfile") >/dev/null 2>&1
+    return $?
+  else
+    return 1
+  fi
+}
+
+function sqoop_extra_classpath_set {
+  if [[ -n "${SQOOP_SERVER_EXTRA_LIB}" ]]; then
+    for f in $SQOOP_SERVER_EXTRA_LIB/*.jar; do
+      CLASSPATH="${CLASSPATH}:$f"
+    done
+  fi
+}
+
 if [ $# = 0 ]; then
   print_usage
   exit
@@ -44,25 +118,30 @@ done
 
 BASEDIR=`dirname ${PRG}`
 BASEDIR=`cd ${BASEDIR}/..;pwd`
+SQOOP_IDENT_STRING=${SQOOP_IDENT_STRING:-$USER}
+SQOOP_PID_DIR=${SQOOP_PID_DIR:-/tmp}
+sqoop_pidfile="${SQOOP_PID_DIR}/sqoop-${SQOOP_IDENT_STRING}-jetty-server.pid"
+JAVA_OPTS="$JAVA_OPTS -Dsqoop.config.dir=`dirname $0`/../conf"
 
 echo "Sqoop home directory: ${BASEDIR}"
 
-CATALINA_BIN=${CATALINA_BIN:-${BASEDIR}/server/bin}
-CLIENT_LIB=${CLIENT_LIB:-${BASEDIR}/shell/lib}
+SQOOP_CLIENT_LIB=${BASEDIR}/shell/lib
+SQOOP_SERVER_LIB=${BASEDIR}/server/lib
+SQOOP_TOOLS_LIB=${BASEDIR}/tools/lib
 
-setup_catalina_opts() {
-  # The Java System properties 'sqoop.http.port' and 'sqoop.admin.port' are
-  # not used by Sqoop. They are used in Tomcat's server.xml configuration file
-  echo "Using   CATALINA_OPTS:       ${CATALINA_OPTS}"
+EXEC_JAVA='java'
+if [ -n "${JAVA_HOME}" ] ; then
+    EXEC_JAVA="${JAVA_HOME}/bin/java"
+fi
 
-  catalina_opts="-Dsqoop.http.port=${SQOOP_HTTP_PORT}";
-  catalina_opts="${catalina_opts} -Dsqoop.admin.port=${SQOOP_ADMIN_PORT}";
-
-  echo "Adding to CATALINA_OPTS:    ${catalina_opts}"
-
-  export CATALINA_OPTS="${CATALINA_OPTS} ${catalina_opts}"
-}
+# validation the java command
+${EXEC_JAVA} -version 2>/dev/null
+if [[ $? -gt 0 ]]; then
+  echo "Can't find the path for java, please check the environment setting."
+  exit
+fi
 
+sqoop_extra_classpath_set
 COMMAND=$1
 case $COMMAND in
   tool)
@@ -72,48 +151,82 @@ case $COMMAND in
     fi
 
     source ${BASEDIR}/bin/sqoop-sys.sh
-    setup_catalina_opts
 
     # Remove the "tool" keyword from the command line and pass the rest
     shift
 
-    $CATALINA_BIN/tool-wrapper.sh -server org.apache.sqoop.tomcat.TomcatToolRunner $@
+    # Build class path with full path to each library,including tools ,server and hadoop related
+    for f in $SQOOP_TOOLS_LIB/*.jar; do
+      CLASSPATH="${CLASSPATH}:$f"
+    done
+
+    # Build class path with full path to each library, including hadoop related
+    sqoop_server_classpath_set
+
+    ${EXEC_JAVA} $JAVA_OPTS -classpath ${CLASSPATH} org.apache.sqoop.tools.ToolRunner $@
     ;;
   server)
     if [ $# = 1 ]; then
       echo "Usage: sqoop.sh server <start/stop>"
       exit
     fi
-    actionCmd=$2
 
     source ${BASEDIR}/bin/sqoop-sys.sh
-    setup_catalina_opts
 
-    # There seems to be a bug in catalina.sh whereby catalina.sh doesn't respect
-    # CATALINA_OPTS when stopping the tomcat server. Consequently, we have to hack around
-    # by specifying the CATALINA_OPTS properties in JAVA_OPTS variable
-    if [ "$actionCmd" == "stop" ]; then
-      export JAVA_OPTS="$JAVA_OPTS $CATALINA_OPTS"
-    fi
+    case $2 in
+      start)
+        # check if the sqoop server started already.
+        is_sqoop_server_running
+        if [[ $? -eq 0 ]]; then
+          echo "The Sqoop server is already started."
+          exit
+        fi
 
-    # Remove the first 2 command line arguments (server and action command (start/stop)) so we can pass
-    # the rest to catalina.sh script
-    shift
-    shift
+        # Build class path with full path to each library, including hadoop related
+        sqoop_server_classpath_set
 
-    $CATALINA_BIN/catalina.sh $actionCmd "$@"
+        echo "Starting the Sqoop2 server..."
+        ${EXEC_JAVA} $JAVA_OPTS -classpath ${CLASSPATH} org.apache.sqoop.server.SqoopJettyServer &
+
+        echo $! > "${sqoop_pidfile}" 2>/dev/null
+        if [[ $? -gt 0 ]]; then
+          echo "ERROR:  Cannot write pid ${pidfile}."
+        fi
+
+        # wait 5 seconds, then check if the sqoop server started successfully.
+        sleep 5
+        is_sqoop_server_running
+        if [[ $? -eq 0 ]]; then
+          echo "Sqoop2 server started."
+        fi
+      ;;
+      stop)
+        # check if the sqoop server stopped already.
+        is_sqoop_server_running
+        if [[ $? -gt 0 ]]; then
+          echo "No Sqoop server is running."
+          exit
+        fi
+
+        pid=$(cat "$sqoop_pidfile")
+        echo "Stopping the Sqoop2 server..."
+        kill -9 "${pid}" >/dev/null 2>&1
+        rm -f "${sqoop_pidfile}"
+        echo "Sqoop2 server stopped."
+      ;;
+      *)
+        echo "Unknown command, usage: sqoop.sh server <start/stop>"
+        exit
+      ;;
+    esac
     ;;
 
   client)
     # Build class path with full path to each library
-    for f in $CLIENT_LIB/*.jar; do
+    for f in $SQOOP_CLIENT_LIB/*.jar; do
       CLASSPATH="${CLASSPATH}:$f"
     done
 
-    EXEC_JAVA='java'
-    if [ -n "${JAVA_HOME}" ] ; then
-        EXEC_JAVA="${JAVA_HOME}/bin/java"
-    fi
     ${EXEC_JAVA} $JAVA_OPTS -classpath ${CLASSPATH} org.apache.sqoop.shell.SqoopShell $2
     ;;
 

dist/src/main/conf/sqoop.properties

@@ -0,0 +1,189 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Sqoop configuration file used by the built in configuration
+# provider: org.apache.sqoop.core.PropertiesConfigurationProvider.
+# This file must reside in the system configuration directory
+# which is specified by the system property "sqoop.config.dir"
+# and must be called sqoop.properties.
+#
+# NOTE: Tokens specified in this file that are marked by a
+# leading and trailing '@' characters should be replaced by
+# their appropriate values. For example, the token @LOGDIR@
+# should be replaced  appropriately.
+#
+# The following tokens are used in this configuration file:
+#
+# LOGDIR
+#   The absolute path to the directory where system genearated
+#   log files will be kept.
+#
+# BASEDIR
+#   The absolute path to the directory where Sqoop 2 is installed
+#
+
+#
+# Logging Configuration
+# Any property that starts with the prefix
+# org.apache.sqoop.log4j is parsed out by the configuration
+# system and passed to the log4j subsystem. This allows you
+# to specify log4j configuration properties from within the
+# Sqoop configuration.
+#
+org.apache.sqoop.log4j.appender.file=org.apache.log4j.RollingFileAppender
+org.apache.sqoop.log4j.appender.file.File=@LOGDIR@/sqoop.log
+org.apache.sqoop.log4j.appender.file.MaxFileSize=25MB
+org.apache.sqoop.log4j.appender.file.MaxBackupIndex=5
+org.apache.sqoop.log4j.appender.file.layout=org.apache.log4j.PatternLayout
+org.apache.sqoop.log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} [%l] %m%n
+org.apache.sqoop.log4j.debug=false
+org.apache.sqoop.log4j.rootCategory=WARN, file
+org.apache.sqoop.log4j.category.org.apache.sqoop=DEBUG
+org.apache.sqoop.log4j.category.org.apache.derby=INFO
+
+#
+# Audit Loggers Configuration
+# Multiple audit loggers could be given here. To specify an
+# audit logger, you should at least add org.apache.sqoop.
+# auditlogger.[LoggerName].class. You could also provide
+# more configuration options by using org.apache.sqoop.
+# auditlogger.[LoggerName] prefix, then all these options
+# are parsed to the logger class.
+#
+org.apache.sqoop.auditlogger.default.class=org.apache.sqoop.audit.FileAuditLogger
+org.apache.sqoop.auditlogger.default.file=@LOGDIR@/default.audit
+
+#
+# Repository configuration
+# The Repository subsystem provides the special prefix which
+# is "org.apache.sqoop.repository.sysprop". Any property that
+# is specified with this prefix is parsed out and set as a
+# system property. For example, if the built in Derby repository
+# is being used, the sysprop prefixed properties can be used
+# to affect Derby configuration at startup time by setting
+# the appropriate system properties.
+#
+
+# Repository provider
+org.apache.sqoop.repository.provider=org.apache.sqoop.repository.JdbcRepositoryProvider
+
+# Repository upgrade
+# If set to true, it will not upgrade the sqoop respository schema, by default it will iniate the upgrade on server start-up
+org.apache.sqoop.repository.schema.immutable=false
+
+# JDBC repository provider configuration
+org.apache.sqoop.repository.jdbc.handler=org.apache.sqoop.repository.derby.DerbyRepositoryHandler
+org.apache.sqoop.repository.jdbc.transaction.isolation=READ_COMMITTED
+org.apache.sqoop.repository.jdbc.maximum.connections=10
+org.apache.sqoop.repository.jdbc.url=jdbc:derby:@BASEDIR@/repository/db;create=true
+org.apache.sqoop.repository.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
+org.apache.sqoop.repository.jdbc.user=sa
+org.apache.sqoop.repository.jdbc.password=
+
+# System properties for embedded Derby configuration
+org.apache.sqoop.repository.sysprop.derby.stream.error.file=@LOGDIR@/derbyrepo.log
+
+#
+# Sqoop Connector configuration
+# If set to true will initiate Connectors config upgrade during server startup
+#
+org.apache.sqoop.connector.autoupgrade=false
+
+#
+# Sqoop Driver configuration
+# If set to true will initiate the Driver config upgrade during server startup
+#
+org.apache.sqoop.driver.autoupgrade=false
+
+# Sleeping period for reloading configuration file (once a minute)
+org.apache.sqoop.core.configuration.provider.properties.sleep=60000
+
+#
+# Submission engine configuration
+#
+
+# Submission engine class
+org.apache.sqoop.submission.engine=org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine
+
+# Number of milliseconds, submissions created before this limit will be removed, default is one day
+#org.apache.sqoop.submission.purge.threshold=
+
+# Number of milliseconds for purge thread to sleep, by default one day
+#org.apache.sqoop.submission.purge.sleep=
+
+# Number of milliseconds for update thread to sleep, by default 5 minutes
+#org.apache.sqoop.submission.update.sleep=
+
+#
+# Configuration for Mapreduce submission engine (applicable if it's configured)
+#
+
+# Hadoop configuration directory
+org.apache.sqoop.submission.engine.mapreduce.configuration.directory=/etc/hadoop/conf/
+
+#
+# Execution engine configuration
+#
+org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.MapreduceExecutionEngine
+
+#
+# Authentication configuration
+#
+#org.apache.sqoop.security.authentication.type=SIMPLE
+#org.apache.sqoop.security.authentication.handler=org.apache.sqoop.security.authentication.SimpleAuthenticationHandler
+#org.apache.sqoop.security.authentication.anonymous=true
+#org.apache.sqoop.security.authentication.type=KERBEROS
+#org.apache.sqoop.security.authentication.handler=org.apache.sqoop.security.authentication.KerberosAuthenticationHandler
+#org.apache.sqoop.security.authentication.kerberos.principal=sqoop/_HOST@NOVALOCAL
+#org.apache.sqoop.security.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab
+#org.apache.sqoop.security.authentication.kerberos.http.principal=HTTP/_HOST@NOVALOCAL
+#org.apache.sqoop.security.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab
+#org.apache.sqoop.security.authentication.enable.doAs=true
+#org.apache.sqoop.security.authentication.proxyuser.#USER#.users=*
+#org.apache.sqoop.security.authentication.proxyuser.#USER#.groups=*
+#org.apache.sqoop.security.authentication.proxyuser.#USER#.hosts=*
+
+# Default user, default value is "sqoop.anonymous.user"
+#org.apache.sqoop.security.authentication.default.user=
+
+#
+# Authorization configuration
+#
+#org.apache.sqoop.security.authorization.handler=org.apache.sqoop.security.authorization.DefaultAuthorizationHandler
+#org.apache.sqoop.security.authorization.access_controller=org.apache.sqoop.security.authorization.DefaultAuthorizationAccessController
+#org.apache.sqoop.security.authorization.validator=org.apache.sqoop.security.authorization.DefaultAuthorizationValidator
+#org.apache.sqoop.security.authorization.authentication_provider=org.apache.sqoop.security.authorization.DefaultAuthenticationProvider
+#org.apache.sqoop.security.authorization.server_name=SqoopServer1
+
+# External connectors load path
+# "/path/to/external/connectors/": Add all the connector JARs in the specified folder
+#
+org.apache.sqoop.connector.external.loadpath=
+
+# Sqoop application classpath
+# ":" separated list of jars to be included in sqoop.
+#
+org.apache.sqoop.classpath.extra=
+
+#
+# Jetty Server configuration
+#
+#org.apache.sqoop.jetty.thread.pool.worker.max=500
+#org.apache.sqoop.jetty.thread.pool.worker.min=5
+#org.apache.sqoop.jetty.thread.pool.worker.alive.time=60
+#org.apache.sqoop.jetty.port=12000

dist/src/main/conf/sqoop_bootstrap.properties

@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Bootstrap configuration for Sqoop. This file is picked up
+# from the directory specified by the system property
+# "sqoop.config.dir". Sqoop will not boot up if this property
+# is not set, or if there is no file by the name
+# "sqoop_bootstrap.properties" in the directory pointed by
+# this system property.
+#
+
+#
+# Specifies the configuration provider to be used.
+# This is a required configuration that must be specified.
+# The default value for this is:
+#   org.apache.sqoop.core.PropertiesConfigurationProvider
+# The PropertiesConfigurationProvider expects the system
+# configuration directory to be specified by the system
+# property "sqoop.config.dir", and it must contain a file
+# by the name "sqoop.properties".
+#
+sqoop.config.provider=org.apache.sqoop.core.PropertiesConfigurationProvider

pom.xml

@@ -123,6 +123,7 @@ limitations under the License.
     <avro.version>1.7.7</avro.version>
     <jcommander.version>1.27</jcommander.version>
     <findbugs.version>1.3.2</findbugs.version>
+    <jetty.version>9.2.13.v20150730</jetty.version>
   </properties>
 
   <dependencies>
@@ -289,7 +290,6 @@ limitations under the License.
       <dependency>
         <groupId>org.apache.sqoop</groupId>
         <artifactId>sqoop-server</artifactId>
-        <type>war</type>
         <version>${project.version}</version>
       </dependency>
       <dependency>
@@ -656,6 +656,16 @@ limitations under the License.
         <version>${findbugs.version}</version>
         <scope>provided</scope>
       </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-server</artifactId>
+        <version>${jetty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-servlet</artifactId>
+        <version>${jetty.version}</version>
+      </dependency>
    </dependencies>
   </dependencyManagement>
 

server/pom.xml

@@ -30,7 +30,6 @@ limitations under the License.
   <groupId>org.apache.sqoop</groupId>
   <artifactId>sqoop-server</artifactId>
   <name>Sqoop Server</name>
-  <packaging>war</packaging>
 
   <dependencies>
     <dependency>
@@ -114,14 +113,6 @@ limitations under the License.
       <scope>provided</scope>
     </dependency>
 
-<!--
-    <dependency>
-      <groupId>org.apache.sqoop.connector</groupId>
-      <artifactId>sqoop-connector-mysql-jdbc</artifactId>
-      <version>2.0.0-SNAPSHOT</version>
-    </dependency>
--->
-
     <dependency>
       <groupId>javax.servlet</groupId>
       <artifactId>servlet-api</artifactId>
@@ -132,10 +123,44 @@ limitations under the License.
       <groupId>com.googlecode.json-simple</groupId>
       <artifactId>json-simple</artifactId>
     </dependency>
+    <dependency>
+       <groupId>org.eclipse.jetty</groupId>
+       <artifactId>jetty-server</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlet</artifactId>
+    </dependency>
   </dependencies>
 
-  <build>
-    <finalName>sqoop</finalName>
-  </build>
+  <profiles>
+    <profile>
+      <id>binary</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <executions>
+              <execution>
+                <phase>package</phase>
+                <goals>
+                    <goal>copy-dependencies</goal>
+                </goals>
+                <configuration>
+                    <includeScope>runtime</includeScope>
+                    <excludeScope>test</excludeScope>
+                    <outputDirectory>${project.build.directory}/lib</outputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 
 </project>

server/src/main/java/org/apache/sqoop/server/SqoopJettyConstants.java

@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.server;
+
+import org.apache.sqoop.core.ConfigurationConstants;
+
+public class SqoopJettyConstants {
+
+  /**
+   * All audit logger related configuration is prefixed with this:
+   * <tt>org.apache.sqoop.jetty.</tt>
+   */
+  public static final String PREFIX_JETTY_CONFIG =
+          ConfigurationConstants.PREFIX_GLOBAL_CONFIG + "jetty.";
+
+  /**
+   * Max number of worker in thread pool to be used, specified by:
+   * <tt>org.apache.sqoop.jetty.thread.pool.worker.max</tt>
+   */
+  public static final String SYSCFG_JETTY_THREAD_POOL_MAX_WORKER = PREFIX_JETTY_CONFIG
+          + "thread.pool.worker.max";
+
+  public static final int DEFAULT_JETTY_THREAD_POOL_MAX_WORKER = 500;
+
+  /**
+   * Min number of worker in thread pool to be used, specified by:
+   * <tt>org.apache.sqoop.jetty.thread.pool.worker.min</tt>
+   */
+  public static final String SYSCFG_JETTY_THREAD_POOL_MIN_WORKER = PREFIX_JETTY_CONFIG
+          + "thread.pool.worker.min";
+
+  public static final int DEFAULT_JETTY_THREAD_POOL_MIN_WORKER = 5;
+
+  /**
+   * Alive time of Worker in thread pool to be used, specified by:
+   * <tt>org.apache.sqoop.jetty.thread.pool.worker.alive.time</tt>
+   */
+  public static final String SYSCFG_JETTY_THREAD_POOL_WORKER_ALIVE_TIME = PREFIX_JETTY_CONFIG
+          + "thread.pool.worker.alive.time";
+
+  public static final long DEFAULT_JETTY_THREAD_POOL_WORKER_ALIVE_TIME = 60;
+
+  /**
+   * The Port number for Jetty server:
+   * <tt>org.apache.sqoop.jetty.port</tt>
+   */
+  public static final String SYSCFG_JETTY_PORT = PREFIX_JETTY_CONFIG + "port";
+
+  public static final int DEFAULT_SYSCFG_JETTY_PORT = 12000;
+}

server/src/main/java/org/apache/sqoop/server/SqoopJettyContext.java

@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.server;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.MapContext;
+
+public class SqoopJettyContext {
+  private static final Logger LOG =
+          Logger.getLogger(SqoopJettyContext.class);
+
+  private final MapContext context;
+  private final int minWorkerThreads;
+  private final int maxWorkerThreads;
+  private final int port;
+  private final long workerKeepAliveTime;
+
+  public SqoopJettyContext(MapContext context) {
+    this.context = context;
+    port = context.getInt(SqoopJettyConstants.SYSCFG_JETTY_PORT,
+            SqoopJettyConstants.DEFAULT_SYSCFG_JETTY_PORT);
+    minWorkerThreads = context.getInt(SqoopJettyConstants.SYSCFG_JETTY_THREAD_POOL_MIN_WORKER,
+            SqoopJettyConstants.DEFAULT_JETTY_THREAD_POOL_MIN_WORKER);
+    maxWorkerThreads = context.getInt(SqoopJettyConstants.SYSCFG_JETTY_THREAD_POOL_MAX_WORKER,
+            SqoopJettyConstants.DEFAULT_JETTY_THREAD_POOL_MAX_WORKER);
+    workerKeepAliveTime = context.getLong(SqoopJettyConstants.SYSCFG_JETTY_THREAD_POOL_WORKER_ALIVE_TIME,
+            SqoopJettyConstants.DEFAULT_JETTY_THREAD_POOL_WORKER_ALIVE_TIME);
+  }
+
+  public int getMinWorkerThreads() {
+    return minWorkerThreads;
+  }
+
+  public int getMaxWorkerThreads() {
+    return maxWorkerThreads;
+  }
+
+  public int getPort() {
+    return port;
+  }
+
+  public long getWorkerKeepAliveTime() {
+    return workerKeepAliveTime;
+  }
+}

server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java

@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.server;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.core.SqoopConfiguration;
+import org.apache.sqoop.core.SqoopServer;
+import org.apache.sqoop.filter.SqoopAuthenticationFilter;
+import org.apache.sqoop.server.v1.*;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.util.thread.ExecutorThreadPool;
+import org.eclipse.jetty.server.ServerConnector;
+
+import javax.servlet.DispatcherType;
+import java.util.EnumSet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Create a Jetty embedded server to answer http requests. The primary goal is
+ * to serve up status information for the server. There are three contexts:
+ * "/logs/" -> points to the log directory "/static/" -> points to common static
+ * files (src/webapps/static) "/" -> the jsp server code from
+ * (src/webapps/<name>)
+ */
+public class SqoopJettyServer {
+  private static final Logger LOG = Logger.getLogger(SqoopJettyServer.class);
+  private Server webServer;
+
+  public SqoopJettyServer() {
+    SqoopServer.initialize();
+    SqoopJettyContext sqoopJettyContext = new SqoopJettyContext(SqoopConfiguration.getInstance().getContext());
+    // Server thread pool
+    // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
+    ExecutorService executorService = new ThreadPoolExecutor(sqoopJettyContext.getMinWorkerThreads(),
+            sqoopJettyContext.getMaxWorkerThreads(),
+            sqoopJettyContext.getWorkerKeepAliveTime(), TimeUnit.SECONDS, new SynchronousQueue<Runnable>());
+    ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService);
+    webServer = new Server(threadPool);
+
+    // Connector configs
+    ServerConnector connector = new ServerConnector(webServer);
+    connector.setPort(sqoopJettyContext.getPort());
+    webServer.addConnector(connector);
+    webServer.setHandler(createServletContextHandler());
+  }
+
+  public synchronized void startServer() {
+    try {
+      webServer.start();
+      LOG.info("Started Sqoop Jetty server.");
+    } catch (Exception e) {
+      LOG.error(e.getMessage(), e);
+      throw new RuntimeException("Sqoop server failed to start.", e);
+    }
+  }
+
+  public synchronized void joinServerThread() {
+    try {
+      webServer.join();
+    } catch (InterruptedException ie) {
+      LOG.info("Sqoop Jetty server is interrupted.");
+    }
+  }
+
+  // this method is only for test
+  public synchronized void stopServerForTest() {
+    try {
+      if (webServer != null && webServer.isStarted()) {
+        webServer.stop();
+        SqoopServer.destroy();
+        LOG.info("Stopped Sqoop Jetty server.");
+      }
+    } catch (Exception e) {
+      LOG.error(e.getMessage(), e);
+    }
+  }
+
+  // this method is only for test
+  public String getServerUrl() {
+    return webServer.getURI().toString() + "/";
+  }
+
+  private static ServletContextHandler createServletContextHandler() {
+    ServletContextHandler context = new ServletContextHandler();
+    context.setContextPath("/sqoop");
+    context.addServlet(AuthorizationServlet.class, "/v1/authorization/*");
+    context.addServlet(ConfigurableServlet.class, "/v1/configurable/*");
+    context.addServlet(ConnectorServlet.class, "/v1/connector/*");
+    context.addServlet(DriverServlet.class, "/v1/driver/*");
+    context.addServlet(JobServlet.class, "/v1/job/*");
+    context.addServlet(JobsServlet.class, "/v1/jobs/*");
+    context.addServlet(LinkServlet.class, "/v1/link/*");
+    context.addServlet(LinksServlet.class, "/v1/links/*");
+    context.addServlet(SubmissionsServlet.class, "/v1/submissions/*");
+    context.addServlet(VersionServlet.class, "/version");
+    context.addFilter(SqoopAuthenticationFilter.class, "/*", EnumSet.allOf(DispatcherType.class));
+    return context;
+  }
+
+  public static void main(String[] args) {
+    SqoopJettyServer sqoopJettyServer = new SqoopJettyServer();
+    sqoopJettyServer.startServer();
+    sqoopJettyServer.joinServerThread();
+  }
+}

test/pom.xml

@@ -85,6 +85,11 @@ limitations under the License.
       <artifactId>sqoop-connector-kite</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.sqoop</groupId>
+      <artifactId>sqoop-server</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.sqoop.connector</groupId>
       <artifactId>sqoop-connector-kafka</artifactId>
@@ -239,7 +244,6 @@ limitations under the License.
           </execution>
         </executions>
       </plugin>
-
     </plugins>
   </build>
 

test/src/main/java/org/apache/sqoop/test/infrastructure/providers/SqoopInfrastructureProvider.java

@@ -19,8 +19,8 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.log4j.Logger;
+import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
 import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
-import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster;
 
 /**
  * Sqoop infrastructure provider.
@@ -37,7 +37,7 @@ public SqoopInfrastructureProvider() {}
   @Override
   public void start() {
     try {
-      instance = new TomcatSqoopMiniCluster(rootPath, hadoopConf);
+      instance = new JettySqoopMiniCluster(rootPath, hadoopConf);
       instance.start();
     } catch (Exception e) {
       LOG.error("Could not start Sqoop mini cluster.", e);

test/src/main/java/org/apache/sqoop/test/minicluster/JettySqoopMiniCluster.java

@@ -0,0 +1,64 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package org.apache.sqoop.test.minicluster;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.server.SqoopJettyServer;
+
+/**
+* Embedded jetty Sqoop server mini cluster.
+*
+* This mini cluster will start up embedded jetty
+*/
+public class JettySqoopMiniCluster extends SqoopMiniCluster {
+
+  private SqoopJettyServer sqoopJettyServer;
+
+  /** {@inheritDoc} */
+  public JettySqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception {
+    super(temporaryPath, configuration);
+  }
+
+  @Override
+  public void start() throws Exception {
+    prepareTemporaryPath();
+    sqoopJettyServer = new SqoopJettyServer();
+    sqoopJettyServer.startServer();
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public void stop() throws Exception {
+    if (sqoopJettyServer != null) {
+      sqoopJettyServer.stopServerForTest();
+    }
+  }
+
+  /**
+  * Return server URL.
+  */
+  @Override
+  public String getServerUrl() {
+    if (sqoopJettyServer != null) {
+      return sqoopJettyServer.getServerUrl();
+    }
+    throw new RuntimeException("Jetty server wasn't started.");
+  }
+}

test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java

@@ -47,7 +47,8 @@
  * In addition to pure Tomcat based test case it will also create and initialize
  * the database provider prior every test execution.
  */
-abstract public class ConnectorTestCase extends TomcatTestCase {
+@edu.umd.cs.findbugs.annotations.SuppressWarnings({"MS_PKGPROTECT", "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD"})
+abstract public class ConnectorTestCase extends JettyTestCase {
 
   private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class);
 
@@ -74,14 +75,14 @@ public void finished(MSubmission submission) {
   };
 
   @BeforeSuite(alwaysRun = true)
-  public static void startProvider() throws Exception {
+  public void startProvider() throws Exception {
     provider = DatabaseProviderFactory.getProvider(System.getProperties());
     LOG.info("Starting database provider: " + provider.getClass().getName());
     provider.start();
   }
 
   @AfterSuite(alwaysRun = true)
-  public static void stopProvider() {
+  public void stopProvider() {
     LOG.info("Stopping database provider: " + provider.getClass().getName());
     provider.stop();
   }

test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java

@@ -30,8 +30,8 @@
 import org.apache.sqoop.test.hadoop.HadoopMiniClusterRunner;
 import org.apache.sqoop.test.hadoop.HadoopRunner;
 import org.apache.sqoop.test.hadoop.HadoopRunnerFactory;
+import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
 import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
-import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster;
 import org.apache.sqoop.test.utils.HdfsUtils;
 import org.testng.ITest;
 import org.testng.ITestContext;
@@ -40,11 +40,12 @@
 import org.testng.annotations.BeforeSuite;
 
 /**
- * Basic test case that will bootstrap Sqoop server running in external Tomcat
+ * Basic test case that will bootstrap Sqoop server running in embedded Jetty
  * process.
  */
-abstract public class TomcatTestCase implements ITest {
-  private static final Logger LOG = Logger.getLogger(TomcatTestCase.class);
+@edu.umd.cs.findbugs.annotations.SuppressWarnings({"MS_PKGPROTECT", "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD"})
+abstract public class JettyTestCase implements ITest {
+  private static final Logger LOG = Logger.getLogger(JettyTestCase.class);
 
   public String methodName;
 
@@ -79,9 +80,9 @@ abstract public class TomcatTestCase implements ITest {
   protected static FileSystem hdfsClient;
 
   /**
-   * Tomcat based Sqoop mini cluster
+   * Jetty based Sqoop mini cluster
    */
-  private static TomcatSqoopMiniCluster cluster;
+  private static JettySqoopMiniCluster cluster;
 
   /**
    * Sqoop client API.
@@ -151,8 +152,8 @@ protected void stopHadoop() throws Exception {
    *
    * @return New instance of test mini cluster
    */
-  public TomcatSqoopMiniCluster createSqoopMiniCluster() throws Exception {
-    return new TomcatSqoopMiniCluster(getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration());
+  public JettySqoopMiniCluster createSqoopMiniCluster() throws Exception {
+    return new JettySqoopMiniCluster(getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration());
   }
 
   /**
@@ -172,7 +173,7 @@ public static SqoopMiniCluster getCluster() {
     return cluster;
   }
 
-  public static void setCluster(TomcatSqoopMiniCluster sqoopMiniClusterluster) {
+  public static void setCluster(JettySqoopMiniCluster sqoopMiniClusterluster) {
     cluster = sqoopMiniClusterluster;
   }
 

test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/DerbyRepositoryUpgradeTest.java

@@ -19,8 +19,8 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.sqoop.client.SqoopClient;
-import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster;
-import org.apache.sqoop.test.testcases.TomcatTestCase;
+import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
+import org.apache.sqoop.test.testcases.JettyTestCase;
 import org.apache.sqoop.test.utils.CompressionUtils;
 import org.apache.sqoop.test.utils.HdfsUtils;
 import org.testng.ITestContext;
@@ -51,14 +51,14 @@
  * methods describing content of the repository (what links/jobs it have, ...).
  *
  */
-public abstract class DerbyRepositoryUpgradeTest extends TomcatTestCase {
+public abstract class DerbyRepositoryUpgradeTest extends JettyTestCase {
 
   private static final Logger LOG = Logger.getLogger(DerbyRepositoryUpgradeTest.class);
 
   /**
    * Custom Sqoop mini cluster that points derby repository to real on-disk structures.
    */
-  public static class DerbySqoopMiniCluster extends TomcatSqoopMiniCluster {
+  public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster {
     private String repositoryPath;
 
     public DerbySqoopMiniCluster(String repositoryPath, String temporaryPath, Configuration configuration) throws Exception {
@@ -66,6 +66,7 @@ public DerbySqoopMiniCluster(String repositoryPath, String temporaryPath, Config
       this.repositoryPath = repositoryPath;
     }
 
+    @Override
     protected Map<String, String> getRepositoryConfiguration() {
       Map<String, String> properties = new HashMap<String, String>();
 
@@ -124,10 +125,10 @@ protected Map<String, String> getRepositoryConfiguration() {
   public abstract Integer[] getDeleteJobIds();
 
   public String getRepositoryPath() {
-    return HdfsUtils.joinPathFragments(getTemporaryTomcatPath(), "repo");
+    return HdfsUtils.joinPathFragments(getTemporaryJettyPath(), "repo");
   }
 
-  public String getTemporaryTomcatPath() {
+  public String getTemporaryJettyPath() {
     return HdfsUtils.joinPathFragments(getTemporaryPath(), getClass().getCanonicalName(), getTestName());
   }
 
@@ -149,7 +150,7 @@ public void startSqoopMiniCluster(ITestContext context) throws Exception {
     CompressionUtils.untarStreamToDirectory(tarballStream, getRepositoryPath());
 
     // And use them for new Derby repo instance
-    setCluster(new DerbySqoopMiniCluster(getRepositoryPath(), getTemporaryTomcatPath() + "/sqoop-mini-cluster", hadoopCluster.getConfiguration()));
+    setCluster(new DerbySqoopMiniCluster(getRepositoryPath(), getTemporaryJettyPath() + "/sqoop-mini-cluster", hadoopCluster.getConfiguration()));
 
     // Start server
     getCluster().start();

tools/pom.xml

@@ -47,5 +47,33 @@ limitations under the License.
       <artifactId>commons-io</artifactId>
     </dependency>
   </dependencies>
-
+  <profiles>
+    <profile>
+      <id>binary</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <executions>
+              <execution>
+                <phase>package</phase>
+                <goals>
+                  <goal>copy-dependencies</goal>
+                </goals>
+                <configuration>
+                  <includeScope>runtime</includeScope>
+                  <excludeScope>test</excludeScope>
+                  <outputDirectory>${project.build.directory}/lib</outputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 </project>