diff --git a/core/src/main/java/org/apache/sqoop/audit/AuditLoggerManager.java b/core/src/main/java/org/apache/sqoop/audit/AuditLoggerManager.java index af9282c2..5e4adc65 100644 --- a/core/src/main/java/org/apache/sqoop/audit/AuditLoggerManager.java +++ b/core/src/main/java/org/apache/sqoop/audit/AuditLoggerManager.java @@ -64,11 +64,13 @@ public static AuditLoggerManager getInstance() { } private AuditLoggerManager() { - loggers = new ArrayList(); } public synchronized void initialize() { LOG.info("Begin audit logger manager initialization"); + if (loggers == null) { + loggers = new ArrayList(); + } initializeLoggers(); SqoopConfiguration.getInstance().getProvider() diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java index b0a68411..285e8938 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java @@ -94,11 +94,11 @@ public static void setInstance(ConnectorManager newInstance) { } // key: connector id, value: connector name - private Map idToNameMap = new HashMap(); + private Map idToNameMap; private Set connectorNames = new HashSet(); // key: connector name, value: connector handler - private Map handlerMap = new HashMap(); + private Map handlerMap; public List getConnectorConfigurables() { List connectors = new LinkedList(); @@ -112,10 +112,6 @@ public Set getConnectorIds() { return idToNameMap.keySet(); } - public Set getConnectorNames() { - return connectorNames; - } - public Map getResourceBundles(Locale locale) { Map bundles = new HashMap(); for (ConnectorHandler handler : handlerMap.values()) { @@ -166,6 +162,16 @@ public synchronized void initialize() { } public synchronized void initialize(boolean autoUpgrade) { + if (handlerMap == null) { + handlerMap = new HashMap(); + } + if (idToNameMap == null) { + idToNameMap = new HashMap(); + } + if (connectorNames == null) { + connectorNames = new HashSet(); + } + if (LOG.isTraceEnabled()) { LOG.trace("Begin connector manager initialization"); } @@ -234,8 +240,9 @@ private synchronized void registerConnectors(boolean autoUpgrade) { } public synchronized void destroy() { - handlerMap = null; - idToNameMap = null; + handlerMap = null; + idToNameMap = null; + connectorNames = null; } @Override diff --git a/core/src/main/java/org/apache/sqoop/core/SqoopServer.java b/core/src/main/java/org/apache/sqoop/core/SqoopServer.java index 75cea655..555728c1 100644 --- a/core/src/main/java/org/apache/sqoop/core/SqoopServer.java +++ b/core/src/main/java/org/apache/sqoop/core/SqoopServer.java @@ -49,7 +49,7 @@ public static void destroy() { public static void initialize() { try { - LOG.info("Booting up Sqoop server"); + LOG.info("Initializing Sqoop server."); SqoopConfiguration.getInstance().initialize(); AuthenticationManager.getInstance().initialize(); AuthorizationManager.getInstance().initialize(); @@ -58,9 +58,9 @@ public static void initialize() { ConnectorManager.getInstance().initialize(); Driver.getInstance().initialize(); JobManager.getInstance().initialize(); - LOG.info("Sqoop server has successfully boot up"); + LOG.info("Sqoop server has successfully been initialized."); } catch (RuntimeException | ClassNotFoundException | IllegalAccessException | InstantiationException e) { - LOG.error("Server startup failure", e); + LOG.error("Failure in server initialization", e); throw new RuntimeException("Failure in server initialization", e); } } diff --git a/core/src/main/java/org/apache/sqoop/driver/JobManager.java b/core/src/main/java/org/apache/sqoop/driver/JobManager.java index de1b9764..ebb7efdc 100644 --- a/core/src/main/java/org/apache/sqoop/driver/JobManager.java +++ b/core/src/main/java/org/apache/sqoop/driver/JobManager.java @@ -138,7 +138,7 @@ public static void setInstance(JobManager newInstance) { /** * Synchronization variable between threads. */ - private boolean running = true; + private boolean running; /** * Specifies how old submissions should be removed from repository. @@ -215,7 +215,7 @@ public synchronized void destroy() { public synchronized void initialize() { LOG.trace("Begin submission engine manager initialization"); MapContext context = SqoopConfiguration.getInstance().getContext(); - + running = true; // Let's load configured submission engine String submissionEngineClassName = context.getString(DriverConstants.SYSCFG_SUBMISSION_ENGINE); diff --git a/dist/pom.xml b/dist/pom.xml index 37cd5baf..5e324a93 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -40,7 +40,6 @@ limitations under the License. org.apache.sqoop sqoop-server - war org.apache.sqoop @@ -155,27 +154,13 @@ limitations under the License. - - - - - - + + + + - - - - - - - - - - - + @@ -188,6 +173,20 @@ limitations under the License. + + + + + + + + + + + + + diff --git a/dist/src/main/bin/sqoop-sys.sh b/dist/src/main/bin/sqoop-sys.sh index 64dd0bf9..194c227f 100644 --- a/dist/src/main/bin/sqoop-sys.sh +++ b/dist/src/main/bin/sqoop-sys.sh @@ -27,17 +27,3 @@ ECHO="true" if [ "${1}" == "-silent" ]; then ECHO="false" fi - -if [ "${SQOOP_HTTP_PORT}" = "" ]; then - export SQOOP_HTTP_PORT=12000 - print "Setting SQOOP_HTTP_PORT: ${SQOOP_HTTP_PORT}" -else - print "Using SQOOP_HTTP_PORT: ${SQOOP_HTTP_PORT}" -fi - -if [ "${SQOOP_ADMIN_PORT}" = "" ]; then - export SQOOP_ADMIN_PORT=`expr $SQOOP_HTTP_PORT + 1` - print "Setting SQOOP_ADMIN_PORT: ${SQOOP_ADMIN_PORT}" -else - print "Using SQOOP_ADMIN_PORT: ${SQOOP_ADMIN_PORT}" -fi diff --git a/dist/src/main/bin/sqoop.sh b/dist/src/main/bin/sqoop.sh index 707c3fc3..1bc046f8 100755 --- a/dist/src/main/bin/sqoop.sh +++ b/dist/src/main/bin/sqoop.sh @@ -24,6 +24,80 @@ function print_usage(){ echo "" } +function sqoop_server_classpath_set { + + HADOOP_COMMON_HOME=${HADOOP_COMMON_HOME:-${HADOOP_HOME}/share/hadoop/common} + HADOOP_HDFS_HOME=${HADOOP_HDFS_HOME:-${HADOOP_HOME}/share/hadoop/hdfs} + HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-${HADOOP_HOME}/share/hadoop/mapreduce} + HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-${HADOOP_HOME}/share/hadoop/yarn} + + if [[ ! (-d "${HADOOP_COMMON_HOME}" && -d "${HADOOP_HDFS_HOME}" && -d "${HADOOP_MAPRED_HOME}" && -d "${HADOOP_YARN_HOME}") ]]; then + echo "Can't load the Hadoop related java lib, please check the setting for the following environment variables:" + echo " HADOOP_COMMON_HOME, HADOOP_HDFS_HOME, HADOOP_MAPRED_HOME, HADOOP_YARN_HOME" + exit + fi + + for f in $SQOOP_SERVER_LIB/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + + for f in $HADOOP_COMMON_HOME/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + + for f in $HADOOP_COMMON_HOME/lib/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + + for f in $HADOOP_HDFS_HOME/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + + for f in $HADOOP_HDFS_HOME/lib/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + + for f in $HADOOP_MAPRED_HOME/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + + for f in $HADOOP_MAPRED_HOME/lib/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + + for f in $HADOOP_YARN_HOME/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + + for f in $HADOOP_YARN_HOME/lib/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + + for f in $HIVE_HOME/lib/*.jar; do + # exclude the jdbc for derby, to avoid the sealing violation exception + if [[ ! $f =~ derby* && ! $f =~ jetty* ]]; then + CLASSPATH="${CLASSPATH}:$f" + fi + done +} + +function is_sqoop_server_running { + if [[ -f "${sqoop_pidfile}" ]]; then + kill -s 0 $(cat "$sqoop_pidfile") >/dev/null 2>&1 + return $? + else + return 1 + fi +} + +function sqoop_extra_classpath_set { + if [[ -n "${SQOOP_SERVER_EXTRA_LIB}" ]]; then + for f in $SQOOP_SERVER_EXTRA_LIB/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + fi +} + if [ $# = 0 ]; then print_usage exit @@ -44,25 +118,30 @@ done BASEDIR=`dirname ${PRG}` BASEDIR=`cd ${BASEDIR}/..;pwd` +SQOOP_IDENT_STRING=${SQOOP_IDENT_STRING:-$USER} +SQOOP_PID_DIR=${SQOOP_PID_DIR:-/tmp} +sqoop_pidfile="${SQOOP_PID_DIR}/sqoop-${SQOOP_IDENT_STRING}-jetty-server.pid" +JAVA_OPTS="$JAVA_OPTS -Dsqoop.config.dir=`dirname $0`/../conf" echo "Sqoop home directory: ${BASEDIR}" -CATALINA_BIN=${CATALINA_BIN:-${BASEDIR}/server/bin} -CLIENT_LIB=${CLIENT_LIB:-${BASEDIR}/shell/lib} +SQOOP_CLIENT_LIB=${BASEDIR}/shell/lib +SQOOP_SERVER_LIB=${BASEDIR}/server/lib +SQOOP_TOOLS_LIB=${BASEDIR}/tools/lib -setup_catalina_opts() { - # The Java System properties 'sqoop.http.port' and 'sqoop.admin.port' are - # not used by Sqoop. They are used in Tomcat's server.xml configuration file - echo "Using CATALINA_OPTS: ${CATALINA_OPTS}" +EXEC_JAVA='java' +if [ -n "${JAVA_HOME}" ] ; then + EXEC_JAVA="${JAVA_HOME}/bin/java" +fi - catalina_opts="-Dsqoop.http.port=${SQOOP_HTTP_PORT}"; - catalina_opts="${catalina_opts} -Dsqoop.admin.port=${SQOOP_ADMIN_PORT}"; - - echo "Adding to CATALINA_OPTS: ${catalina_opts}" - - export CATALINA_OPTS="${CATALINA_OPTS} ${catalina_opts}" -} +# validation the java command +${EXEC_JAVA} -version 2>/dev/null +if [[ $? -gt 0 ]]; then + echo "Can't find the path for java, please check the environment setting." + exit +fi +sqoop_extra_classpath_set COMMAND=$1 case $COMMAND in tool) @@ -72,48 +151,82 @@ case $COMMAND in fi source ${BASEDIR}/bin/sqoop-sys.sh - setup_catalina_opts # Remove the "tool" keyword from the command line and pass the rest shift - $CATALINA_BIN/tool-wrapper.sh -server org.apache.sqoop.tomcat.TomcatToolRunner $@ + # Build class path with full path to each library,including tools ,server and hadoop related + for f in $SQOOP_TOOLS_LIB/*.jar; do + CLASSPATH="${CLASSPATH}:$f" + done + + # Build class path with full path to each library, including hadoop related + sqoop_server_classpath_set + + ${EXEC_JAVA} $JAVA_OPTS -classpath ${CLASSPATH} org.apache.sqoop.tools.ToolRunner $@ ;; server) if [ $# = 1 ]; then echo "Usage: sqoop.sh server " exit fi - actionCmd=$2 source ${BASEDIR}/bin/sqoop-sys.sh - setup_catalina_opts - # There seems to be a bug in catalina.sh whereby catalina.sh doesn't respect - # CATALINA_OPTS when stopping the tomcat server. Consequently, we have to hack around - # by specifying the CATALINA_OPTS properties in JAVA_OPTS variable - if [ "$actionCmd" == "stop" ]; then - export JAVA_OPTS="$JAVA_OPTS $CATALINA_OPTS" - fi + case $2 in + start) + # check if the sqoop server started already. + is_sqoop_server_running + if [[ $? -eq 0 ]]; then + echo "The Sqoop server is already started." + exit + fi - # Remove the first 2 command line arguments (server and action command (start/stop)) so we can pass - # the rest to catalina.sh script - shift - shift + # Build class path with full path to each library, including hadoop related + sqoop_server_classpath_set - $CATALINA_BIN/catalina.sh $actionCmd "$@" + echo "Starting the Sqoop2 server..." + ${EXEC_JAVA} $JAVA_OPTS -classpath ${CLASSPATH} org.apache.sqoop.server.SqoopJettyServer & + + echo $! > "${sqoop_pidfile}" 2>/dev/null + if [[ $? -gt 0 ]]; then + echo "ERROR: Cannot write pid ${pidfile}." + fi + + # wait 5 seconds, then check if the sqoop server started successfully. + sleep 5 + is_sqoop_server_running + if [[ $? -eq 0 ]]; then + echo "Sqoop2 server started." + fi + ;; + stop) + # check if the sqoop server stopped already. + is_sqoop_server_running + if [[ $? -gt 0 ]]; then + echo "No Sqoop server is running." + exit + fi + + pid=$(cat "$sqoop_pidfile") + echo "Stopping the Sqoop2 server..." + kill -9 "${pid}" >/dev/null 2>&1 + rm -f "${sqoop_pidfile}" + echo "Sqoop2 server stopped." + ;; + *) + echo "Unknown command, usage: sqoop.sh server " + exit + ;; + esac ;; client) # Build class path with full path to each library - for f in $CLIENT_LIB/*.jar; do + for f in $SQOOP_CLIENT_LIB/*.jar; do CLASSPATH="${CLASSPATH}:$f" done - EXEC_JAVA='java' - if [ -n "${JAVA_HOME}" ] ; then - EXEC_JAVA="${JAVA_HOME}/bin/java" - fi ${EXEC_JAVA} $JAVA_OPTS -classpath ${CLASSPATH} org.apache.sqoop.shell.SqoopShell $2 ;; diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties new file mode 100755 index 00000000..f077ecc7 --- /dev/null +++ b/dist/src/main/conf/sqoop.properties @@ -0,0 +1,189 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Sqoop configuration file used by the built in configuration +# provider: org.apache.sqoop.core.PropertiesConfigurationProvider. +# This file must reside in the system configuration directory +# which is specified by the system property "sqoop.config.dir" +# and must be called sqoop.properties. +# +# NOTE: Tokens specified in this file that are marked by a +# leading and trailing '@' characters should be replaced by +# their appropriate values. For example, the token @LOGDIR@ +# should be replaced appropriately. +# +# The following tokens are used in this configuration file: +# +# LOGDIR +# The absolute path to the directory where system genearated +# log files will be kept. +# +# BASEDIR +# The absolute path to the directory where Sqoop 2 is installed +# + +# +# Logging Configuration +# Any property that starts with the prefix +# org.apache.sqoop.log4j is parsed out by the configuration +# system and passed to the log4j subsystem. This allows you +# to specify log4j configuration properties from within the +# Sqoop configuration. +# +org.apache.sqoop.log4j.appender.file=org.apache.log4j.RollingFileAppender +org.apache.sqoop.log4j.appender.file.File=@LOGDIR@/sqoop.log +org.apache.sqoop.log4j.appender.file.MaxFileSize=25MB +org.apache.sqoop.log4j.appender.file.MaxBackupIndex=5 +org.apache.sqoop.log4j.appender.file.layout=org.apache.log4j.PatternLayout +org.apache.sqoop.log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} [%l] %m%n +org.apache.sqoop.log4j.debug=false +org.apache.sqoop.log4j.rootCategory=WARN, file +org.apache.sqoop.log4j.category.org.apache.sqoop=DEBUG +org.apache.sqoop.log4j.category.org.apache.derby=INFO + +# +# Audit Loggers Configuration +# Multiple audit loggers could be given here. To specify an +# audit logger, you should at least add org.apache.sqoop. +# auditlogger.[LoggerName].class. You could also provide +# more configuration options by using org.apache.sqoop. +# auditlogger.[LoggerName] prefix, then all these options +# are parsed to the logger class. +# +org.apache.sqoop.auditlogger.default.class=org.apache.sqoop.audit.FileAuditLogger +org.apache.sqoop.auditlogger.default.file=@LOGDIR@/default.audit + +# +# Repository configuration +# The Repository subsystem provides the special prefix which +# is "org.apache.sqoop.repository.sysprop". Any property that +# is specified with this prefix is parsed out and set as a +# system property. For example, if the built in Derby repository +# is being used, the sysprop prefixed properties can be used +# to affect Derby configuration at startup time by setting +# the appropriate system properties. +# + +# Repository provider +org.apache.sqoop.repository.provider=org.apache.sqoop.repository.JdbcRepositoryProvider + +# Repository upgrade +# If set to true, it will not upgrade the sqoop respository schema, by default it will iniate the upgrade on server start-up +org.apache.sqoop.repository.schema.immutable=false + +# JDBC repository provider configuration +org.apache.sqoop.repository.jdbc.handler=org.apache.sqoop.repository.derby.DerbyRepositoryHandler +org.apache.sqoop.repository.jdbc.transaction.isolation=READ_COMMITTED +org.apache.sqoop.repository.jdbc.maximum.connections=10 +org.apache.sqoop.repository.jdbc.url=jdbc:derby:@BASEDIR@/repository/db;create=true +org.apache.sqoop.repository.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver +org.apache.sqoop.repository.jdbc.user=sa +org.apache.sqoop.repository.jdbc.password= + +# System properties for embedded Derby configuration +org.apache.sqoop.repository.sysprop.derby.stream.error.file=@LOGDIR@/derbyrepo.log + +# +# Sqoop Connector configuration +# If set to true will initiate Connectors config upgrade during server startup +# +org.apache.sqoop.connector.autoupgrade=false + +# +# Sqoop Driver configuration +# If set to true will initiate the Driver config upgrade during server startup +# +org.apache.sqoop.driver.autoupgrade=false + +# Sleeping period for reloading configuration file (once a minute) +org.apache.sqoop.core.configuration.provider.properties.sleep=60000 + +# +# Submission engine configuration +# + +# Submission engine class +org.apache.sqoop.submission.engine=org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine + +# Number of milliseconds, submissions created before this limit will be removed, default is one day +#org.apache.sqoop.submission.purge.threshold= + +# Number of milliseconds for purge thread to sleep, by default one day +#org.apache.sqoop.submission.purge.sleep= + +# Number of milliseconds for update thread to sleep, by default 5 minutes +#org.apache.sqoop.submission.update.sleep= + +# +# Configuration for Mapreduce submission engine (applicable if it's configured) +# + +# Hadoop configuration directory +org.apache.sqoop.submission.engine.mapreduce.configuration.directory=/etc/hadoop/conf/ + +# +# Execution engine configuration +# +org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.MapreduceExecutionEngine + +# +# Authentication configuration +# +#org.apache.sqoop.security.authentication.type=SIMPLE +#org.apache.sqoop.security.authentication.handler=org.apache.sqoop.security.authentication.SimpleAuthenticationHandler +#org.apache.sqoop.security.authentication.anonymous=true +#org.apache.sqoop.security.authentication.type=KERBEROS +#org.apache.sqoop.security.authentication.handler=org.apache.sqoop.security.authentication.KerberosAuthenticationHandler +#org.apache.sqoop.security.authentication.kerberos.principal=sqoop/_HOST@NOVALOCAL +#org.apache.sqoop.security.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab +#org.apache.sqoop.security.authentication.kerberos.http.principal=HTTP/_HOST@NOVALOCAL +#org.apache.sqoop.security.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab +#org.apache.sqoop.security.authentication.enable.doAs=true +#org.apache.sqoop.security.authentication.proxyuser.#USER#.users=* +#org.apache.sqoop.security.authentication.proxyuser.#USER#.groups=* +#org.apache.sqoop.security.authentication.proxyuser.#USER#.hosts=* + +# Default user, default value is "sqoop.anonymous.user" +#org.apache.sqoop.security.authentication.default.user= + +# +# Authorization configuration +# +#org.apache.sqoop.security.authorization.handler=org.apache.sqoop.security.authorization.DefaultAuthorizationHandler +#org.apache.sqoop.security.authorization.access_controller=org.apache.sqoop.security.authorization.DefaultAuthorizationAccessController +#org.apache.sqoop.security.authorization.validator=org.apache.sqoop.security.authorization.DefaultAuthorizationValidator +#org.apache.sqoop.security.authorization.authentication_provider=org.apache.sqoop.security.authorization.DefaultAuthenticationProvider +#org.apache.sqoop.security.authorization.server_name=SqoopServer1 + +# External connectors load path +# "/path/to/external/connectors/": Add all the connector JARs in the specified folder +# +org.apache.sqoop.connector.external.loadpath= + +# Sqoop application classpath +# ":" separated list of jars to be included in sqoop. +# +org.apache.sqoop.classpath.extra= + +# +# Jetty Server configuration +# +#org.apache.sqoop.jetty.thread.pool.worker.max=500 +#org.apache.sqoop.jetty.thread.pool.worker.min=5 +#org.apache.sqoop.jetty.thread.pool.worker.alive.time=60 +#org.apache.sqoop.jetty.port=12000 diff --git a/dist/src/main/conf/sqoop_bootstrap.properties b/dist/src/main/conf/sqoop_bootstrap.properties new file mode 100755 index 00000000..d7bfb5ec --- /dev/null +++ b/dist/src/main/conf/sqoop_bootstrap.properties @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Bootstrap configuration for Sqoop. This file is picked up +# from the directory specified by the system property +# "sqoop.config.dir". Sqoop will not boot up if this property +# is not set, or if there is no file by the name +# "sqoop_bootstrap.properties" in the directory pointed by +# this system property. +# + +# +# Specifies the configuration provider to be used. +# This is a required configuration that must be specified. +# The default value for this is: +# org.apache.sqoop.core.PropertiesConfigurationProvider +# The PropertiesConfigurationProvider expects the system +# configuration directory to be specified by the system +# property "sqoop.config.dir", and it must contain a file +# by the name "sqoop.properties". +# +sqoop.config.provider=org.apache.sqoop.core.PropertiesConfigurationProvider \ No newline at end of file diff --git a/pom.xml b/pom.xml index ef3f5f43..e4381134 100644 --- a/pom.xml +++ b/pom.xml @@ -123,6 +123,7 @@ limitations under the License. 1.7.7 1.27 1.3.2 + 9.2.13.v20150730 @@ -289,7 +290,6 @@ limitations under the License. org.apache.sqoop sqoop-server - war ${project.version} @@ -656,6 +656,16 @@ limitations under the License. ${findbugs.version} provided + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + diff --git a/server/pom.xml b/server/pom.xml index 59663faf..ca068e04 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -30,7 +30,6 @@ limitations under the License. org.apache.sqoop sqoop-server Sqoop Server - war @@ -114,14 +113,6 @@ limitations under the License. provided - - javax.servlet servlet-api @@ -132,10 +123,44 @@ limitations under the License. com.googlecode.json-simple json-simple + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + - - sqoop - + + + binary + + false + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + package + + copy-dependencies + + + runtime + test + ${project.build.directory}/lib + + + + + + + + diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyConstants.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyConstants.java new file mode 100644 index 00000000..cc2d604c --- /dev/null +++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyConstants.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.server; + +import org.apache.sqoop.core.ConfigurationConstants; + +public class SqoopJettyConstants { + + /** + * All audit logger related configuration is prefixed with this: + * org.apache.sqoop.jetty. + */ + public static final String PREFIX_JETTY_CONFIG = + ConfigurationConstants.PREFIX_GLOBAL_CONFIG + "jetty."; + + /** + * Max number of worker in thread pool to be used, specified by: + * org.apache.sqoop.jetty.thread.pool.worker.max + */ + public static final String SYSCFG_JETTY_THREAD_POOL_MAX_WORKER = PREFIX_JETTY_CONFIG + + "thread.pool.worker.max"; + + public static final int DEFAULT_JETTY_THREAD_POOL_MAX_WORKER = 500; + + /** + * Min number of worker in thread pool to be used, specified by: + * org.apache.sqoop.jetty.thread.pool.worker.min + */ + public static final String SYSCFG_JETTY_THREAD_POOL_MIN_WORKER = PREFIX_JETTY_CONFIG + + "thread.pool.worker.min"; + + public static final int DEFAULT_JETTY_THREAD_POOL_MIN_WORKER = 5; + + /** + * Alive time of Worker in thread pool to be used, specified by: + * org.apache.sqoop.jetty.thread.pool.worker.alive.time + */ + public static final String SYSCFG_JETTY_THREAD_POOL_WORKER_ALIVE_TIME = PREFIX_JETTY_CONFIG + + "thread.pool.worker.alive.time"; + + public static final long DEFAULT_JETTY_THREAD_POOL_WORKER_ALIVE_TIME = 60; + + /** + * The Port number for Jetty server: + * org.apache.sqoop.jetty.port + */ + public static final String SYSCFG_JETTY_PORT = PREFIX_JETTY_CONFIG + "port"; + + public static final int DEFAULT_SYSCFG_JETTY_PORT = 12000; +} diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyContext.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyContext.java new file mode 100644 index 00000000..9bb3b978 --- /dev/null +++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyContext.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.server; + +import org.apache.log4j.Logger; +import org.apache.sqoop.common.MapContext; + +public class SqoopJettyContext { + private static final Logger LOG = + Logger.getLogger(SqoopJettyContext.class); + + private final MapContext context; + private final int minWorkerThreads; + private final int maxWorkerThreads; + private final int port; + private final long workerKeepAliveTime; + + public SqoopJettyContext(MapContext context) { + this.context = context; + port = context.getInt(SqoopJettyConstants.SYSCFG_JETTY_PORT, + SqoopJettyConstants.DEFAULT_SYSCFG_JETTY_PORT); + minWorkerThreads = context.getInt(SqoopJettyConstants.SYSCFG_JETTY_THREAD_POOL_MIN_WORKER, + SqoopJettyConstants.DEFAULT_JETTY_THREAD_POOL_MIN_WORKER); + maxWorkerThreads = context.getInt(SqoopJettyConstants.SYSCFG_JETTY_THREAD_POOL_MAX_WORKER, + SqoopJettyConstants.DEFAULT_JETTY_THREAD_POOL_MAX_WORKER); + workerKeepAliveTime = context.getLong(SqoopJettyConstants.SYSCFG_JETTY_THREAD_POOL_WORKER_ALIVE_TIME, + SqoopJettyConstants.DEFAULT_JETTY_THREAD_POOL_WORKER_ALIVE_TIME); + } + + public int getMinWorkerThreads() { + return minWorkerThreads; + } + + public int getMaxWorkerThreads() { + return maxWorkerThreads; + } + + public int getPort() { + return port; + } + + public long getWorkerKeepAliveTime() { + return workerKeepAliveTime; + } +} diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java new file mode 100644 index 00000000..3c46b1a2 --- /dev/null +++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.server; + +import org.apache.log4j.Logger; +import org.apache.sqoop.core.SqoopConfiguration; +import org.apache.sqoop.core.SqoopServer; +import org.apache.sqoop.filter.SqoopAuthenticationFilter; +import org.apache.sqoop.server.v1.*; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.util.thread.ExecutorThreadPool; +import org.eclipse.jetty.server.ServerConnector; + +import javax.servlet.DispatcherType; +import java.util.EnumSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Create a Jetty embedded server to answer http requests. The primary goal is + * to serve up status information for the server. There are three contexts: + * "/logs/" -> points to the log directory "/static/" -> points to common static + * files (src/webapps/static) "/" -> the jsp server code from + * (src/webapps/) + */ +public class SqoopJettyServer { + private static final Logger LOG = Logger.getLogger(SqoopJettyServer.class); + private Server webServer; + + public SqoopJettyServer() { + SqoopServer.initialize(); + SqoopJettyContext sqoopJettyContext = new SqoopJettyContext(SqoopConfiguration.getInstance().getContext()); + // Server thread pool + // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests + ExecutorService executorService = new ThreadPoolExecutor(sqoopJettyContext.getMinWorkerThreads(), + sqoopJettyContext.getMaxWorkerThreads(), + sqoopJettyContext.getWorkerKeepAliveTime(), TimeUnit.SECONDS, new SynchronousQueue()); + ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService); + webServer = new Server(threadPool); + + // Connector configs + ServerConnector connector = new ServerConnector(webServer); + connector.setPort(sqoopJettyContext.getPort()); + webServer.addConnector(connector); + webServer.setHandler(createServletContextHandler()); + } + + public synchronized void startServer() { + try { + webServer.start(); + LOG.info("Started Sqoop Jetty server."); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException("Sqoop server failed to start.", e); + } + } + + public synchronized void joinServerThread() { + try { + webServer.join(); + } catch (InterruptedException ie) { + LOG.info("Sqoop Jetty server is interrupted."); + } + } + + // this method is only for test + public synchronized void stopServerForTest() { + try { + if (webServer != null && webServer.isStarted()) { + webServer.stop(); + SqoopServer.destroy(); + LOG.info("Stopped Sqoop Jetty server."); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + // this method is only for test + public String getServerUrl() { + return webServer.getURI().toString() + "/"; + } + + private static ServletContextHandler createServletContextHandler() { + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/sqoop"); + context.addServlet(AuthorizationServlet.class, "/v1/authorization/*"); + context.addServlet(ConfigurableServlet.class, "/v1/configurable/*"); + context.addServlet(ConnectorServlet.class, "/v1/connector/*"); + context.addServlet(DriverServlet.class, "/v1/driver/*"); + context.addServlet(JobServlet.class, "/v1/job/*"); + context.addServlet(JobsServlet.class, "/v1/jobs/*"); + context.addServlet(LinkServlet.class, "/v1/link/*"); + context.addServlet(LinksServlet.class, "/v1/links/*"); + context.addServlet(SubmissionsServlet.class, "/v1/submissions/*"); + context.addServlet(VersionServlet.class, "/version"); + context.addFilter(SqoopAuthenticationFilter.class, "/*", EnumSet.allOf(DispatcherType.class)); + return context; + } + + public static void main(String[] args) { + SqoopJettyServer sqoopJettyServer = new SqoopJettyServer(); + sqoopJettyServer.startServer(); + sqoopJettyServer.joinServerThread(); + } +} diff --git a/test/pom.xml b/test/pom.xml index 8218477f..54d54f42 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -85,6 +85,11 @@ limitations under the License. sqoop-connector-kite + + org.apache.sqoop + sqoop-server + + org.apache.sqoop.connector sqoop-connector-kafka @@ -239,7 +244,6 @@ limitations under the License. - diff --git a/test/src/main/java/org/apache/sqoop/test/infrastructure/providers/SqoopInfrastructureProvider.java b/test/src/main/java/org/apache/sqoop/test/infrastructure/providers/SqoopInfrastructureProvider.java index 5b4f5950..9b0d5217 100644 --- a/test/src/main/java/org/apache/sqoop/test/infrastructure/providers/SqoopInfrastructureProvider.java +++ b/test/src/main/java/org/apache/sqoop/test/infrastructure/providers/SqoopInfrastructureProvider.java @@ -19,8 +19,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; import org.apache.sqoop.test.minicluster.SqoopMiniCluster; -import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster; /** * Sqoop infrastructure provider. @@ -37,7 +37,7 @@ public SqoopInfrastructureProvider() {} @Override public void start() { try { - instance = new TomcatSqoopMiniCluster(rootPath, hadoopConf); + instance = new JettySqoopMiniCluster(rootPath, hadoopConf); instance.start(); } catch (Exception e) { LOG.error("Could not start Sqoop mini cluster.", e); diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/JettySqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/JettySqoopMiniCluster.java new file mode 100644 index 00000000..325a790c --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/JettySqoopMiniCluster.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.test.minicluster; + +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; +import org.apache.sqoop.server.SqoopJettyServer; + +/** +* Embedded jetty Sqoop server mini cluster. +* +* This mini cluster will start up embedded jetty +*/ +public class JettySqoopMiniCluster extends SqoopMiniCluster { + + private SqoopJettyServer sqoopJettyServer; + + /** {@inheritDoc} */ + public JettySqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception { + super(temporaryPath, configuration); + } + + @Override + public void start() throws Exception { + prepareTemporaryPath(); + sqoopJettyServer = new SqoopJettyServer(); + sqoopJettyServer.startServer(); + } + + /** {@inheritDoc} */ + @Override + public void stop() throws Exception { + if (sqoopJettyServer != null) { + sqoopJettyServer.stopServerForTest(); + } + } + + /** + * Return server URL. + */ + @Override + public String getServerUrl() { + if (sqoopJettyServer != null) { + return sqoopJettyServer.getServerUrl(); + } + throw new RuntimeException("Jetty server wasn't started."); + } +} diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java index 4452558e..5435ade2 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java @@ -47,7 +47,8 @@ * In addition to pure Tomcat based test case it will also create and initialize * the database provider prior every test execution. */ -abstract public class ConnectorTestCase extends TomcatTestCase { +@edu.umd.cs.findbugs.annotations.SuppressWarnings({"MS_PKGPROTECT", "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD"}) +abstract public class ConnectorTestCase extends JettyTestCase { private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class); @@ -74,14 +75,14 @@ public void finished(MSubmission submission) { }; @BeforeSuite(alwaysRun = true) - public static void startProvider() throws Exception { + public void startProvider() throws Exception { provider = DatabaseProviderFactory.getProvider(System.getProperties()); LOG.info("Starting database provider: " + provider.getClass().getName()); provider.start(); } @AfterSuite(alwaysRun = true) - public static void stopProvider() { + public void stopProvider() { LOG.info("Stopping database provider: " + provider.getClass().getName()); provider.stop(); } diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java similarity index 90% rename from test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java rename to test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java index 666749bb..d967aaad 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java @@ -30,8 +30,8 @@ import org.apache.sqoop.test.hadoop.HadoopMiniClusterRunner; import org.apache.sqoop.test.hadoop.HadoopRunner; import org.apache.sqoop.test.hadoop.HadoopRunnerFactory; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; import org.apache.sqoop.test.minicluster.SqoopMiniCluster; -import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster; import org.apache.sqoop.test.utils.HdfsUtils; import org.testng.ITest; import org.testng.ITestContext; @@ -40,11 +40,12 @@ import org.testng.annotations.BeforeSuite; /** - * Basic test case that will bootstrap Sqoop server running in external Tomcat + * Basic test case that will bootstrap Sqoop server running in embedded Jetty * process. */ -abstract public class TomcatTestCase implements ITest { - private static final Logger LOG = Logger.getLogger(TomcatTestCase.class); +@edu.umd.cs.findbugs.annotations.SuppressWarnings({"MS_PKGPROTECT", "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD"}) +abstract public class JettyTestCase implements ITest { + private static final Logger LOG = Logger.getLogger(JettyTestCase.class); public String methodName; @@ -79,9 +80,9 @@ abstract public class TomcatTestCase implements ITest { protected static FileSystem hdfsClient; /** - * Tomcat based Sqoop mini cluster + * Jetty based Sqoop mini cluster */ - private static TomcatSqoopMiniCluster cluster; + private static JettySqoopMiniCluster cluster; /** * Sqoop client API. @@ -151,8 +152,8 @@ protected void stopHadoop() throws Exception { * * @return New instance of test mini cluster */ - public TomcatSqoopMiniCluster createSqoopMiniCluster() throws Exception { - return new TomcatSqoopMiniCluster(getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration()); + public JettySqoopMiniCluster createSqoopMiniCluster() throws Exception { + return new JettySqoopMiniCluster(getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration()); } /** @@ -172,7 +173,7 @@ public static SqoopMiniCluster getCluster() { return cluster; } - public static void setCluster(TomcatSqoopMiniCluster sqoopMiniClusterluster) { + public static void setCluster(JettySqoopMiniCluster sqoopMiniClusterluster) { cluster = sqoopMiniClusterluster; } diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/DerbyRepositoryUpgradeTest.java b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/DerbyRepositoryUpgradeTest.java index 1d5b692a..98e1fa15 100644 --- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/DerbyRepositoryUpgradeTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/DerbyRepositoryUpgradeTest.java @@ -19,8 +19,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.sqoop.client.SqoopClient; -import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster; -import org.apache.sqoop.test.testcases.TomcatTestCase; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; +import org.apache.sqoop.test.testcases.JettyTestCase; import org.apache.sqoop.test.utils.CompressionUtils; import org.apache.sqoop.test.utils.HdfsUtils; import org.testng.ITestContext; @@ -51,14 +51,14 @@ * methods describing content of the repository (what links/jobs it have, ...). * */ -public abstract class DerbyRepositoryUpgradeTest extends TomcatTestCase { +public abstract class DerbyRepositoryUpgradeTest extends JettyTestCase { private static final Logger LOG = Logger.getLogger(DerbyRepositoryUpgradeTest.class); /** * Custom Sqoop mini cluster that points derby repository to real on-disk structures. */ - public static class DerbySqoopMiniCluster extends TomcatSqoopMiniCluster { + public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { private String repositoryPath; public DerbySqoopMiniCluster(String repositoryPath, String temporaryPath, Configuration configuration) throws Exception { @@ -66,6 +66,7 @@ public DerbySqoopMiniCluster(String repositoryPath, String temporaryPath, Config this.repositoryPath = repositoryPath; } + @Override protected Map getRepositoryConfiguration() { Map properties = new HashMap(); @@ -124,10 +125,10 @@ protected Map getRepositoryConfiguration() { public abstract Integer[] getDeleteJobIds(); public String getRepositoryPath() { - return HdfsUtils.joinPathFragments(getTemporaryTomcatPath(), "repo"); + return HdfsUtils.joinPathFragments(getTemporaryJettyPath(), "repo"); } - public String getTemporaryTomcatPath() { + public String getTemporaryJettyPath() { return HdfsUtils.joinPathFragments(getTemporaryPath(), getClass().getCanonicalName(), getTestName()); } @@ -149,7 +150,7 @@ public void startSqoopMiniCluster(ITestContext context) throws Exception { CompressionUtils.untarStreamToDirectory(tarballStream, getRepositoryPath()); // And use them for new Derby repo instance - setCluster(new DerbySqoopMiniCluster(getRepositoryPath(), getTemporaryTomcatPath() + "/sqoop-mini-cluster", hadoopCluster.getConfiguration())); + setCluster(new DerbySqoopMiniCluster(getRepositoryPath(), getTemporaryJettyPath() + "/sqoop-mini-cluster", hadoopCluster.getConfiguration())); // Start server getCluster().start(); diff --git a/tools/pom.xml b/tools/pom.xml index 65fd40ca..0eb56dc8 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -47,5 +47,33 @@ limitations under the License. commons-io - + + + binary + + false + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + package + + copy-dependencies + + + runtime + test + ${project.build.directory}/lib + + + + + + + +