SQOOP-2720: Sqoop2: Allow the integration tests to be run against a real cluster

(Abraham Fine via Jarek Jarcec Cecho)

client/src/main/java/org/apache/sqoop/client/SqoopClient.java

@@ -508,6 +508,23 @@ public void deleteJob(long jobId) {
     resourceRequests.deleteJob(String.valueOf(jobId));
   }
 
+  public void deleteAllLinks(){
+    for (MJob job : getJobs()) {
+      deleteJob(job.getName());
+    }
+  }
+
+  public void deleteAllJobs(){
+    for (MLink link : getLinks()) {
+      deleteLink(link.getName());
+    }
+  }
+
+  public void deleteAllLinksAndJobs(){
+    deleteAllLinks();
+    deleteAllJobs();
+  }
+
   /**
    * Start job with given name.
    *

pom.xml

@@ -230,6 +230,33 @@ limitations under the License.
       </build>
     </profile>
 
+    <profile>
+      <id>realcluster</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <excludedGroups>no-real-cluster</excludedGroups>
+              <reportNameSuffix>real-cluster-integration</reportNameSuffix>
+            </configuration>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <configuration>
+              <excludedGroups>no-real-cluster</excludedGroups>
+              <reportNameSuffix>real-cluster-integration</reportNameSuffix>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
     <!-- Sign profile for releasing artifacts to Nexus repository -->
     <profile>
       <id>sign</id>
@@ -260,10 +287,10 @@ limitations under the License.
         <artifactId>sqoop-client</artifactId>
         <version>${project.version}</version>
       </dependency>
-	  <dependency>
+      <dependency>
-		<groupId>org.apache.sqoop</groupId>
+        <groupId>org.apache.sqoop</groupId>
-		<artifactId>sqoop-shell</artifactId>
+        <artifactId>sqoop-shell</artifactId>
-		<version>${project.version}</version>
+        <version>${project.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.sqoop</groupId>
@@ -500,10 +527,10 @@ limitations under the License.
         <artifactId>commons-cli</artifactId>
         <version>${commons-cli.version}</version>
       </dependency>
-       <dependency>
+      <dependency>
-         <groupId>commons-io</groupId>
+        <groupId>commons-io</groupId>
-         <artifactId>commons-io</artifactId>
+        <artifactId>commons-io</artifactId>
-         <version>${commons-io.version}</version>
+        <version>${commons-io.version}</version>
       </dependency>
       <dependency>
         <groupId>commons-codec</groupId>
@@ -640,9 +667,9 @@ limitations under the License.
         <version>${hadoop.2.version}</version>
       </dependency>
       <dependency>
-       <groupId>org.apache.avro</groupId>
+        <groupId>org.apache.avro</groupId>
-       <artifactId>avro</artifactId>
+        <artifactId>avro</artifactId>
-       <version>${avro.version}</version>
+        <version>${avro.version}</version>
       </dependency>
       <dependency>
         <groupId>net.sourceforge.findbugs</groupId>
@@ -660,7 +687,7 @@ limitations under the License.
         <artifactId>jetty-servlet</artifactId>
         <version>${jetty.version}</version>
       </dependency>
-   </dependencies>
+    </dependencies>
   </dependencyManagement>
 
   <modules>

test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java

@@ -18,6 +18,9 @@
 package org.apache.sqoop.test.minicluster;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.client.SqoopClient;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
 
 /**
  */
@@ -27,11 +30,15 @@ public class RealSqoopCluster extends SqoopMiniCluster {
 
   private String serverUrl;
 
+  private SqoopClient client;
+
   public RealSqoopCluster(String temporaryPath) throws Exception {
     super(temporaryPath);
 
     serverUrl = System.getProperty(SERVER_URL_KEY);
 
+    client = new SqoopClient(serverUrl);
+
     if(serverUrl == null) {
       throw new RuntimeException("Missing URL for real Sqoop 2 server: " + SERVER_URL_KEY);
     }
@@ -44,16 +51,21 @@ public RealSqoopCluster(String temporaryPath, Configuration configuration) throw
 
   @Override
   public void start() throws Exception {
-    // Void operation
+    client.deleteAllLinksAndJobs();
   }
 
   @Override
   public void stop() throws Exception {
-    // Void operation
+    client.deleteAllLinksAndJobs();
   }
 
   @Override
   public String getServerUrl() {
     return serverUrl;
   }
+
+  @Override
+  public String getConfigurationPath() {
+    return "/etc/hadoop/conf/";
+  }
 }

test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java

@@ -17,12 +17,9 @@
  */
 package org.apache.sqoop.test.testcases;
 
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.log4j.Logger;
 import org.apache.sqoop.client.SqoopClient;
@@ -31,7 +28,9 @@
 import org.apache.sqoop.test.hadoop.HadoopRunner;
 import org.apache.sqoop.test.hadoop.HadoopRunnerFactory;
 import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
+import org.apache.sqoop.test.minicluster.RealSqoopCluster;
 import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
+import org.apache.sqoop.test.minicluster.SqoopMiniClusterFactory;
 import org.apache.sqoop.test.utils.HdfsUtils;
 import org.testng.ITest;
 import org.testng.ITestContext;
@@ -39,6 +38,10 @@
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeSuite;
 
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+
 /**
  * Basic test case that will bootstrap Sqoop server running in embedded Jetty
  * process.
@@ -82,7 +85,7 @@ abstract public class JettyTestCase implements ITest {
   /**
    * Jetty based Sqoop mini cluster
    */
-  private static JettySqoopMiniCluster cluster;
+  private static SqoopMiniCluster cluster;
 
   /**
    * Sqoop client API.
@@ -97,8 +100,9 @@ public String getTestName() {
   }
 
   @BeforeMethod(alwaysRun = true)
-  public void setMethodName(Method method) throws Exception {
+  public void setupMehodNameAndOutputPath(Method method) throws Exception {
     methodName = method.getName();
+    hdfsClient.delete(new Path(getMapreduceDirectory()), true);
   }
 
   @BeforeSuite(alwaysRun = true)
@@ -154,8 +158,9 @@ protected void stopHadoop() throws Exception {
    *
    * @return New instance of test mini cluster
    */
-  public JettySqoopMiniCluster createSqoopMiniCluster() throws Exception {
+  public SqoopMiniCluster createSqoopMiniCluster() throws Exception {
-    return new JettySqoopMiniCluster(getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration());
+    return SqoopMiniClusterFactory.getSqoopMiniCluster(System.getProperties(), JettySqoopMiniCluster.class,
+      getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration());
   }
 
   /**

test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java

@@ -37,7 +37,7 @@
 import java.lang.reflect.Method;
 import java.util.List;
 
-@Test(groups = "slow")
+@Test(groups = {"slow", "no-real-cluster"})
 public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase implements ITest {
   private String testName;
 

test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java

@@ -23,7 +23,7 @@
 import org.apache.sqoop.test.testcases.KafkaConnectorTestCase;
 import org.testng.annotations.Test;
 
-
+@Test(groups = "no-real-cluster")
 public class FromHDFSToKafkaTest extends KafkaConnectorTestCase {
 
   public static final String[] input = {

test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java

@@ -25,7 +25,7 @@
 import org.apache.sqoop.test.testcases.KafkaConnectorTestCase;
 import org.testng.annotations.Test;
 
-
+@Test(groups = "no-real-cluster")
 public class FromRDBMSToKafkaTest extends KafkaConnectorTestCase {
 
   private static final String[] input = {

test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java

@@ -30,6 +30,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+@Test(groups = "no-real-cluster")
 public class BlacklistedConnectorTest extends ConnectorTestCase {
   public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster {
     public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception {

test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java

@@ -53,6 +53,7 @@
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
 
+@Test(groups = "no-real-cluster")
 public class ClasspathTest extends ConnectorTestCase {
 
   private static final String TEST_CONNECTOR_JAR_NAME = "test-connector.jar";

test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java

@@ -41,6 +41,7 @@
  * Job with id 3 has been disabled
  * Job with id 1 has been run 5 times
  */
+@Test(groups = "no-real-cluster")
 public class Derby1_99_3UpgradeTest extends DerbyRepositoryUpgradeTest {
 
   @Override

test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java

@@ -17,6 +17,8 @@
  */
 package org.apache.sqoop.integration.repository.derby.upgrade;
 
+import org.testng.annotations.Test;
+
 import java.util.HashMap;
 import java.util.Map;
 
@@ -42,6 +44,7 @@
  * Job with id 4 has been disabled
  * Job with id 5 has been disabled
  */
+@Test(groups = "no-real-cluster")
 public class Derby1_99_4UpgradeTest extends DerbyRepositoryUpgradeTest {
 
   @Override

test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java

@@ -51,6 +51,7 @@
  * Job with id 4 has been disabled
  * Job with id 5 has been disabled
  */
+@Test(groups = "no-real-cluster")
 public class Derby1_99_5UpgradeTest extends DerbyRepositoryUpgradeTest {
 
   @Override

test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java

@@ -52,6 +52,7 @@
  * Job with id 4 has been disabled
  * Job with id 5 has been disabled
  */
+@Test(groups = "no-real-cluster")
 public class Derby1_99_6UpgradeTest extends DerbyRepositoryUpgradeTest {
 
   @Override