From 6c4743de158e4ced43e57703a8bcee5fca6257c6 Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Tue, 8 Dec 2015 08:11:26 +0100 Subject: [PATCH] SQOOP-2720: Sqoop2: Allow the integration tests to be run against a real cluster (Abraham Fine via Jarek Jarcec Cecho) --- .../org/apache/sqoop/client/SqoopClient.java | 17 +++++++ pom.xml | 51 ++++++++++++++----- .../test/minicluster/RealSqoopCluster.java | 16 +++++- .../sqoop/test/testcases/JettyTestCase.java | 21 +++++--- .../hive/FromRDBMSToKiteHiveTest.java | 2 +- .../connector/kafka/FromHDFSToKafkaTest.java | 2 +- .../connector/kafka/FromRDBMSToKafkaTest.java | 2 +- .../BlacklistedConnectorTest.java | 1 + .../connectorloading/ClasspathTest.java | 1 + .../derby/upgrade/Derby1_99_3UpgradeTest.java | 1 + .../derby/upgrade/Derby1_99_4UpgradeTest.java | 3 ++ .../derby/upgrade/Derby1_99_5UpgradeTest.java | 1 + .../derby/upgrade/Derby1_99_6UpgradeTest.java | 1 + 13 files changed, 94 insertions(+), 25 deletions(-) diff --git a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java index 9b9359b2..4abee772 100644 --- a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java +++ b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java @@ -508,6 +508,23 @@ public void deleteJob(long jobId) { resourceRequests.deleteJob(String.valueOf(jobId)); } + public void deleteAllLinks(){ + for (MJob job : getJobs()) { + deleteJob(job.getName()); + } + } + + public void deleteAllJobs(){ + for (MLink link : getLinks()) { + deleteLink(link.getName()); + } + } + + public void deleteAllLinksAndJobs(){ + deleteAllLinks(); + deleteAllJobs(); + } + /** * Start job with given name. * diff --git a/pom.xml b/pom.xml index 91721ce3..e66de4df 100644 --- a/pom.xml +++ b/pom.xml @@ -230,6 +230,33 @@ limitations under the License. + + realcluster + + false + + + + + org.apache.maven.plugins + maven-surefire-plugin + + no-real-cluster + real-cluster-integration + + + + org.apache.maven.plugins + maven-failsafe-plugin + + no-real-cluster + real-cluster-integration + + + + + + sign @@ -260,10 +287,10 @@ limitations under the License. sqoop-client ${project.version} - - org.apache.sqoop - sqoop-shell - ${project.version} + + org.apache.sqoop + sqoop-shell + ${project.version} org.apache.sqoop @@ -500,10 +527,10 @@ limitations under the License. commons-cli ${commons-cli.version} - - commons-io - commons-io - ${commons-io.version} + + commons-io + commons-io + ${commons-io.version} commons-codec @@ -640,9 +667,9 @@ limitations under the License. ${hadoop.2.version} - org.apache.avro - avro - ${avro.version} + org.apache.avro + avro + ${avro.version} net.sourceforge.findbugs @@ -660,7 +687,7 @@ limitations under the License. jetty-servlet ${jetty.version} - + diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java index 325a6a91..cf19730c 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java @@ -18,6 +18,9 @@ package org.apache.sqoop.test.minicluster; import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.client.SqoopClient; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; /** */ @@ -27,11 +30,15 @@ public class RealSqoopCluster extends SqoopMiniCluster { private String serverUrl; + private SqoopClient client; + public RealSqoopCluster(String temporaryPath) throws Exception { super(temporaryPath); serverUrl = System.getProperty(SERVER_URL_KEY); + client = new SqoopClient(serverUrl); + if(serverUrl == null) { throw new RuntimeException("Missing URL for real Sqoop 2 server: " + SERVER_URL_KEY); } @@ -44,16 +51,21 @@ public RealSqoopCluster(String temporaryPath, Configuration configuration) throw @Override public void start() throws Exception { - // Void operation + client.deleteAllLinksAndJobs(); } @Override public void stop() throws Exception { - // Void operation + client.deleteAllLinksAndJobs(); } @Override public String getServerUrl() { return serverUrl; } + + @Override + public String getConfigurationPath() { + return "/etc/hadoop/conf/"; + } } diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java index e78adc0e..e3ee3004 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java @@ -17,12 +17,9 @@ */ package org.apache.sqoop.test.testcases; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Method; - import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; import org.apache.log4j.Logger; import org.apache.sqoop.client.SqoopClient; @@ -31,7 +28,9 @@ import org.apache.sqoop.test.hadoop.HadoopRunner; import org.apache.sqoop.test.hadoop.HadoopRunnerFactory; import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; +import org.apache.sqoop.test.minicluster.RealSqoopCluster; import org.apache.sqoop.test.minicluster.SqoopMiniCluster; +import org.apache.sqoop.test.minicluster.SqoopMiniClusterFactory; import org.apache.sqoop.test.utils.HdfsUtils; import org.testng.ITest; import org.testng.ITestContext; @@ -39,6 +38,10 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeSuite; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; + /** * Basic test case that will bootstrap Sqoop server running in embedded Jetty * process. @@ -82,7 +85,7 @@ abstract public class JettyTestCase implements ITest { /** * Jetty based Sqoop mini cluster */ - private static JettySqoopMiniCluster cluster; + private static SqoopMiniCluster cluster; /** * Sqoop client API. @@ -97,8 +100,9 @@ public String getTestName() { } @BeforeMethod(alwaysRun = true) - public void setMethodName(Method method) throws Exception { + public void setupMehodNameAndOutputPath(Method method) throws Exception { methodName = method.getName(); + hdfsClient.delete(new Path(getMapreduceDirectory()), true); } @BeforeSuite(alwaysRun = true) @@ -154,8 +158,9 @@ protected void stopHadoop() throws Exception { * * @return New instance of test mini cluster */ - public JettySqoopMiniCluster createSqoopMiniCluster() throws Exception { - return new JettySqoopMiniCluster(getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration()); + public SqoopMiniCluster createSqoopMiniCluster() throws Exception { + return SqoopMiniClusterFactory.getSqoopMiniCluster(System.getProperties(), JettySqoopMiniCluster.class, + getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration()); } /** diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java index 0e46bf39..e9644227 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java @@ -37,7 +37,7 @@ import java.lang.reflect.Method; import java.util.List; -@Test(groups = "slow") +@Test(groups = {"slow", "no-real-cluster"}) public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase implements ITest { private String testName; diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java index aa062fb7..ea8c7b9f 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java @@ -23,7 +23,7 @@ import org.apache.sqoop.test.testcases.KafkaConnectorTestCase; import org.testng.annotations.Test; - +@Test(groups = "no-real-cluster") public class FromHDFSToKafkaTest extends KafkaConnectorTestCase { public static final String[] input = { diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java index 6e78a133..4ecb018a 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java @@ -25,7 +25,7 @@ import org.apache.sqoop.test.testcases.KafkaConnectorTestCase; import org.testng.annotations.Test; - +@Test(groups = "no-real-cluster") public class FromRDBMSToKafkaTest extends KafkaConnectorTestCase { private static final String[] input = { diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java index 6228b0d5..8b57d75d 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.Map; +@Test(groups = "no-real-cluster") public class BlacklistedConnectorTest extends ConnectorTestCase { public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception { diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java index 4a2e7a4c..e0879ed8 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java @@ -53,6 +53,7 @@ import java.util.jar.JarOutputStream; import java.util.jar.Manifest; +@Test(groups = "no-real-cluster") public class ClasspathTest extends ConnectorTestCase { private static final String TEST_CONNECTOR_JAR_NAME = "test-connector.jar"; diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java index 8d02e24c..2cfd43cf 100644 --- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java @@ -41,6 +41,7 @@ * Job with id 3 has been disabled * Job with id 1 has been run 5 times */ +@Test(groups = "no-real-cluster") public class Derby1_99_3UpgradeTest extends DerbyRepositoryUpgradeTest { @Override diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java index b88940ac..9f7b04b4 100644 --- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java @@ -17,6 +17,8 @@ */ package org.apache.sqoop.integration.repository.derby.upgrade; +import org.testng.annotations.Test; + import java.util.HashMap; import java.util.Map; @@ -42,6 +44,7 @@ * Job with id 4 has been disabled * Job with id 5 has been disabled */ +@Test(groups = "no-real-cluster") public class Derby1_99_4UpgradeTest extends DerbyRepositoryUpgradeTest { @Override diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java index 1f3563d1..3b1e2146 100644 --- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java @@ -51,6 +51,7 @@ * Job with id 4 has been disabled * Job with id 5 has been disabled */ +@Test(groups = "no-real-cluster") public class Derby1_99_5UpgradeTest extends DerbyRepositoryUpgradeTest { @Override diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java index a57a420a..1081e50a 100644 --- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java @@ -52,6 +52,7 @@ * Job with id 4 has been disabled * Job with id 5 has been disabled */ +@Test(groups = "no-real-cluster") public class Derby1_99_6UpgradeTest extends DerbyRepositoryUpgradeTest { @Override