diff --git a/pom.xml b/pom.xml index 76833b74..ee013f2c 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,7 @@ limitations under the License. 1.6 1.4 2.5 + 2.4 10.8.2.2 1.0.3 2.0.2-alpha @@ -101,6 +102,7 @@ limitations under the License. 4.9 1.2.16 2.5 + 1.3.2 @@ -126,6 +128,13 @@ limitations under the License. + + + commons-io + commons-io + ${commons-io.version} + + com.google.guava guava @@ -138,6 +147,13 @@ limitations under the License. ${hadoop.1.version} provided + + + org.apache.hadoop + hadoop-test + ${hadoop.1.version} + + @@ -181,6 +197,12 @@ limitations under the License. provided + + org.apache.hadoop + hadoop-minicluster + ${hadoop.2.version} + + @@ -310,6 +332,11 @@ limitations under the License. derby ${derby.version} + + org.codehaus.cargo + cargo-core-container-tomcat + ${cargo.version} + @@ -325,6 +352,7 @@ limitations under the License. execution submission dist + test diff --git a/test/pom.xml b/test/pom.xml new file mode 100644 index 00000000..66382b6f --- /dev/null +++ b/test/pom.xml @@ -0,0 +1,213 @@ + + + + 4.0.0 + + + org.apache + sqoop + 2.0.0-SNAPSHOT + + + org.apache.sqoop + test + Sqoop Integration Tests + + + + junit + junit + test + + + + org.apache.sqoop + sqoop-core + + + + org.apache.sqoop + sqoop-client + + + + org.apache.sqoop.execution + sqoop-execution-mapreduce + hadoop${hadoop.profile} + + + + org.apache.sqoop.submission + sqoop-submission-mapreduce + hadoop${hadoop.profile} + + + + org.apache.sqoop.repository + sqoop-repository-derby + + + + org.apache.sqoop.connector + sqoop-connector-generic-jdbc + + + + org.codehaus.cargo + cargo-core-container-tomcat + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + hadoop${hadoop.profile} + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/integration/** + + + + + + integration-test + + test + + integration-test + + + none + + + **/integration/** + + + ${project.build.directory} + + + + + + + + + + + + + + + hadoop100 + + + + hadoop.profile + 100 + + + + + + + commons-io + commons-io + + + + com.google.guava + guava + + + + org.apache.hadoop + hadoop-core + provided + + + + org.apache.hadoop + hadoop-test + + + + + + + + hadoop200 + + + true + + hadoop.profile + 200 + + + + + 200 + + + + + org.apache.hadoop + hadoop-common + provided + + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + provided + + + + org.apache.hadoop + hadoop-minicluster + + + + + + + + diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/InProcessSqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/InProcessSqoopMiniCluster.java new file mode 100644 index 00000000..e11e82c3 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/InProcessSqoopMiniCluster.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.minicluster; + +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.core.SqoopServer; + +/** + * In process Sqoop server mini cluster. + * + * This class will create and initialize Sqoop mini cluster that will be accessible + * using usual manager calls and running inside one single thread. Created server + * won't be accessible over HTTP. + */ +public class InProcessSqoopMiniCluster extends SqoopMiniCluster { + + /** {@inheritDoc} */ + public InProcessSqoopMiniCluster(String temporaryPath) throws Exception { + super(temporaryPath); + } + + /** {@inheritDoc} */ + public InProcessSqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception { + super(temporaryPath, configuration); + } + + /** {@inheritDoc} */ + @Override + public void start() throws Exception { + prepareTemporaryPath(); + SqoopServer.initialize(); + } + + /** {@inheritDoc} */ + @Override + public void stop() throws Exception { + SqoopServer.destroy(); + } + +} diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java new file mode 100644 index 00000000..3620c28a --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.minicluster; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.core.ConfigurationConstants; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Basic tools to bootstrap Sqoop Mini cluster. + */ +public abstract class SqoopMiniCluster { + + /** + * Hadoop configuration. + * + * Either mini cluster generated or real one if we're running on real cluster. + */ + private Configuration configuration; + + /** + * Temporary path for storing Sqoop server data (configuration files) + */ + private String temporaryPath; + + /** + * Create Sqoop Mini cluster with default configuration + * + * @param temporaryPath Temporary path + * @throws Exception + */ + public SqoopMiniCluster(String temporaryPath) throws Exception { + this(temporaryPath, new Configuration()); + } + + /** + * Create Sqoop Mini cluster + * + * @param temporaryPath Temporary path + * @param configuration Hadoop configuration + * @throws Exception + */ + public SqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception { + this.temporaryPath = temporaryPath; + this.configuration = configuration; + } + + /** + * Return temporary path + * + * @return Temporary path + */ + public String getTemporaryPath() { + return temporaryPath; + } + + public String getConfigurationPath() { + return temporaryPath + "/config/"; + } + + public String getLogPath() { + return temporaryPath + "/log/"; + } + + /** + * Start Sqoop Mini cluster + * + * @throws Exception + */ + abstract public void start() throws Exception; + + /** + * Stop Sqoop mini cluster + * + * @throws Exception + */ + abstract public void stop() throws Exception; + + /** + * Prepare temporary directory for starting Sqoop server. + * + * @throws IOException + */ + protected void prepareTemporaryPath() throws IOException { + File tmpDir = new File(getTemporaryPath()); + File configDir = new File(getConfigurationPath()); + File logDir = new File(getLogPath()); + + FileUtils.deleteDirectory(tmpDir); + FileUtils.forceMkdir(tmpDir); + FileUtils.forceMkdir(configDir); + FileUtils.forceMkdir(logDir); + + // Create configuration files + System.setProperty(ConfigurationConstants.SYSPROP_CONFIG_DIR, getConfigurationPath()); + + // sqoop_bootstrap.properties + FileUtils.writeStringToFile(new File(getConfigurationPath() + "sqoop_bootstrap.properties"), "sqoop.config.provider=org.apache.sqoop.core.PropertiesConfigurationProvider"); + + // sqoop.properties + // TODO: This should be generated more dynamically so that user can specify Repository, Submission and Execution engines + File f = new File(getConfigurationPath() + "sqoop.properties"); + + List sqoopProperties = new LinkedList(); + mapToProperties(sqoopProperties, getLoggerConfiguration()); + mapToProperties(sqoopProperties, getRepositoryConfiguration()); + mapToProperties(sqoopProperties, getSubmissionEngineConfiguration()); + mapToProperties(sqoopProperties, getExecutionEngineConfiguration()); + + FileUtils.writeLines(f, sqoopProperties); + + // Hadoop configuration + OutputStream stream = FileUtils.openOutputStream(new File(getConfigurationPath() + "hadoop-site.xml")); + configuration.writeXml(stream); + stream.close(); + } + + private void mapToProperties(List output, Map input) { + for(Map.Entry entry : input.entrySet()) { + output.add(entry.getKey() + "=" + entry.getValue()); + } + } + + /** + * Return properties for logger configuration. + * + * Default implementation will configure server to log into console. + * + * @return + */ + protected Map getLoggerConfiguration() { + Map properties = new HashMap(); + + properties.put("org.apache.sqoop.log4j.appender.file", "org.apache.log4j.ConsoleAppender"); + properties.put("org.apache.sqoop.log4j.appender.file.layout", "org.apache.log4j.PatternLayout"); + properties.put("org.apache.sqoop.log4j.appender.file.layout.ConversionPattern", "%d{ISO8601} %-5p %c{2} [%l] %m%n"); + properties.put("org.apache.sqoop.log4j.debug", "true"); + properties.put("org.apache.sqoop.log4j.rootCategory", "WARN, file"); + properties.put("org.apache.sqoop.log4j.category.org.apache.sqoop", "DEBUG"); + properties.put("org.apache.sqoop.log4j.category.org.apache.derby", "INFO"); + + return properties; + } + + protected Map getRepositoryConfiguration() { + Map properties = new HashMap(); + + properties.put("org.apache.sqoop.repository.provider", "org.apache.sqoop.repository.JdbcRepositoryProvider"); + properties.put("org.apache.sqoop.repository.jdbc.handler", "org.apache.sqoop.repository.derby.DerbyRepositoryHandler"); + properties.put("org.apache.sqoop.repository.jdbc.transaction.isolation", "READ_COMMITTED"); + properties.put("org.apache.sqoop.repository.jdbc.maximum.connections", "10"); + properties.put("org.apache.sqoop.repository.jdbc.url=jdbc:derby:memory:myDB;create", "true"); + properties.put("org.apache.sqoop.repository.jdbc.create.schema", "true"); + properties.put("org.apache.sqoop.repository.jdbc.driver", "org.apache.derby.jdbc.EmbeddedDriver"); + properties.put("org.apache.sqoop.repository.jdbc.user", "sa"); + properties.put("org.apache.sqoop.repository.jdbc.password", ""); + + return properties; + } + + protected Map getSubmissionEngineConfiguration() { + Map properties = new HashMap(); + + properties.put("org.apache.sqoop.submission.engine", "org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine"); + properties.put("org.apache.sqoop.submission.engine.mapreduce.configuration.directory", getConfigurationPath()); + + return properties; + } + + protected Map getExecutionEngineConfiguration() { + Map properties = new HashMap(); + + properties.put("org.apache.sqoop.execution.engine", "org.apache.sqoop.execution.mapreduce.MapreduceExecutionEngine"); + + return properties; + } + +} diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java new file mode 100644 index 00000000..5fa294a2 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.minicluster; + +import org.apache.hadoop.conf.Configuration; +import org.codehaus.cargo.container.ContainerType; +import org.codehaus.cargo.container.InstalledLocalContainer; +import org.codehaus.cargo.container.configuration.ConfigurationType; +import org.codehaus.cargo.container.configuration.LocalConfiguration; +import org.codehaus.cargo.container.deployable.WAR; +import org.codehaus.cargo.container.installer.Installer; +import org.codehaus.cargo.container.installer.ZipURLInstaller; +import org.codehaus.cargo.generic.DefaultContainerFactory; +import org.codehaus.cargo.generic.configuration.DefaultConfigurationFactory; + +import java.net.URL; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Embedded tomcat Sqoop server mini cluster. + * + * This mini cluster will start up embedded tomcat + */ +public class TomcatSqoopMiniCluster extends SqoopMiniCluster { + + private InstalledLocalContainer container = null; + + /** {@inheritDoc} */ + public TomcatSqoopMiniCluster(String temporaryPath) throws Exception { + super(temporaryPath); + } + + /** {@inheritDoc} */ + public TomcatSqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception { + super(temporaryPath, configuration); + } + + /** {@inheritDoc} */ + @Override + public void start() throws Exception { + // Container has already been started + if(container != null) { + return; + } + + prepareTemporaryPath(); + + // TODO(jarcec): We should parametrize those paths, version, etc... + // Source: http://cargo.codehaus.org/Functional+testing + Installer installer = new ZipURLInstaller(new URL("http://archive.apache.org/dist/tomcat/tomcat-6/v6.0.36/bin/apache-tomcat-6.0.36.zip")); + installer.install(); + + LocalConfiguration configuration = (LocalConfiguration) new DefaultConfigurationFactory().createConfiguration("tomcat6x", ContainerType.INSTALLED, ConfigurationType.STANDALONE); + container = (InstalledLocalContainer) new DefaultContainerFactory().createContainer("tomcat6x", ContainerType.INSTALLED, configuration); + + // Set home to our installed tomcat instance + container.setHome(installer.getHome()); + + // Store tomcat logs into file as they are quite handy for debugging + container.setOutput(getTemporaryPath() + "/log/tomcat.log"); + + // Propagate system properties to the container + Map map = new HashMap((Map) System.getProperties()); + container.setSystemProperties(map); + + // Propagate Hadoop jars to the container classpath + // In real world, they would be installed manually by user + List extraClassPath = new LinkedList(); + String []classpath = System.getProperty("java.class.path").split(":"); + for(String jar : classpath) { + System.out.println("JAR: " + jar); + if(jar.contains("hadoop-") || // Hadoop jars + jar.contains("commons-") || // Apache Commons libraries + jar.contains("log4j-") || // Log4j + jar.contains("slf4j-") || // Slf4j + jar.contains("jackson-") || // Jackson + jar.contains("google") // Google libraries (guava, ...) + ) { + extraClassPath.add(jar); + } + } + container.setExtraClasspath(extraClassPath.toArray(new String[extraClassPath.size()])); + + // Finally deploy Sqoop server war file + configuration.addDeployable(new WAR("../server/target/sqoop.war")); + + // Start Sqoop server + container.start(); + } + + /** {@inheritDoc} */ + @Override + public void stop() throws Exception { + container.stop(); + container = null; + } + + + /** + * Return properties for logger configuration. + * + * Tomcat implementation will log into log file instead of console. + * + * @return + */ + protected Map getLoggerConfiguration() { + Map properties = new HashMap(); + + properties.put("org.apache.sqoop.log4j.appender.file", "org.apache.log4j.RollingFileAppender"); + properties.put("org.apache.sqoop.log4j.appender.file.File", getLogPath() + "sqoop.log"); + properties.put("org.apache.sqoop.log4j.appender.file.MaxFileSize", "25MB"); + properties.put("org.apache.sqoop.log4j.appender.file.MaxBackupIndex", "5"); + properties.put("org.apache.sqoop.log4j.appender.file.layout", "org.apache.log4j.PatternLayout"); + properties.put("org.apache.sqoop.log4j.appender.file.layout.ConversionPattern", "%d{ISO8601} %-5p %c{2} [%l] %m%n"); + properties.put("org.apache.sqoop.log4j.debug", "true"); + properties.put("org.apache.sqoop.log4j.rootCategory", "WARN, file"); + properties.put("org.apache.sqoop.log4j.category.org.apache.sqoop", "DEBUG"); + properties.put("org.apache.sqoop.log4j.category.org.apache.derby", "INFO"); + + return properties; + } + + /** + * Return server URL. + */ + public String getServerUrl() { + // We're not doing any changes, so return default URL + return "http://localhost:8080/sqoop/"; + } +} diff --git a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java new file mode 100644 index 00000000..eacf3045 --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.integration; + +import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster; +import org.junit.After; +import org.junit.Before; + +/** + * Basic test case that will bootstrap Sqoop server running in external Tomcat + * process. + */ +abstract public class TomcatTestCase { + + /** + * Temporary path that will be used for this test. + * + * By default we will take look for sqoop.integration.tmpdir property that is + * filled up by maven. If the test is not started from maven (IDE) we will + * pick up configured java.io.tmpdir value. The last results is /tmp/ directory + * in case that no property is set. + */ + private final String TMP_PATH = + System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp")) + + "/sqoop-cargo-tests/" + getClass().getName() + "/"; + + /** + * Tomcat based Sqoop mini cluster + */ + private TomcatSqoopMiniCluster cluster; + + @Before + public void setUp() throws Exception { + cluster = new TomcatSqoopMiniCluster(TMP_PATH); + cluster.start(); + } + + @After + public void cleanUp() throws Exception { + cluster.stop(); + } + + /** + * Return testing server URL + * + * @return + */ + public String getServerUrl() { + return cluster.getServerUrl(); + } +} diff --git a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java new file mode 100644 index 00000000..0f48a8b6 --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.integration.server; + +import org.apache.sqoop.client.request.VersionRequest; +import org.apache.sqoop.common.VersionInfo; +import org.apache.sqoop.integration.TomcatTestCase; +import org.apache.sqoop.json.VersionBean; +import org.junit.Test; + +import static junit.framework.Assert.assertEquals; + +/** + * Basic test to check that server is working and returning correct version info. + */ +public class VersionTest extends TomcatTestCase { + + @Test + public void testVersion() { + VersionRequest versionRequest = new VersionRequest(); + VersionBean versionBean = versionRequest.doGet(getServerUrl()); + + assertEquals(versionBean.getVersion(), VersionInfo.getVersion()); + assertEquals(versionBean.getDate(), VersionInfo.getDate()); + assertEquals(versionBean.getRevision(), VersionInfo.getRevision()); + assertEquals(versionBean.getUser(), VersionInfo.getUser()); + assertEquals(versionBean.getRevision(), VersionInfo.getRevision()); + } + +}