SQOOP-1068: Sqoop2: Add integration tests for real cluster

(Syed Hashmi via Abraham Elmahrek)

test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java

@@ -21,14 +21,15 @@
 import org.apache.sqoop.test.utils.HdfsUtils;
 
 /**
- * Represents a local cluster.
+ * Represents a local cluster. It uses an unchanged Configuration object.
- * It uses an unchanged Configuration object.
+ * HadoopRunner implementation that is using LocalJobRunner for executing
- * HadoopRunner implementation that is using LocalJobRunner for executing mapreduce jobs and local filesystem instead of HDFS.
+ * mapreduce jobs and local filesystem instead of HDFS.
  */
 public class HadoopLocalRunner extends HadoopRunner {
 
   @Override
-  public Configuration prepareConfiguration(Configuration conf) {
+  public Configuration prepareConfiguration(Configuration conf)
+      throws Exception {
     return conf;
   }
 

test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java

@@ -24,13 +24,13 @@
 import org.apache.log4j.Logger;
 
 /**
- * Represents a minicluster setup.
+ * Represents a minicluster setup. It creates a configuration object and mutates
- * It creates a configuration object and mutates it.
+ * it. Clients that need to connect to the miniclusters should use the provided
- * Clients that need to connect to the miniclusters should use
+ * configuration object.
- * the provided configuration object.
  */
 public class HadoopMiniClusterRunner extends HadoopRunner {
-  private static final Logger LOG = Logger.getLogger(HadoopMiniClusterRunner.class);
+  private static final Logger LOG = Logger
+      .getLogger(HadoopMiniClusterRunner.class);
 
   /**
    * Hadoop HDFS cluster
@@ -43,7 +43,8 @@ public class HadoopMiniClusterRunner extends HadoopRunner {
   protected MiniMRCluster mrCluster;
 
   @Override
-  public Configuration prepareConfiguration(Configuration config) {
+  public Configuration prepareConfiguration(Configuration config)
+      throws Exception {
     config.set("dfs.block.access.token.enable", "false");
     config.set("dfs.permissions", "true");
     config.set("hadoop.security.authentication", "simple");
@@ -51,7 +52,8 @@ public Configuration prepareConfiguration(Configuration config) {
     config.set("mapred.tasktracker.reduce.tasks.maximum", "1");
     config.set("mapred.submit.replication", "1");
     config.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
-    config.set("yarn.application.classpath", System.getProperty("java.class.path"));
+    config.set("yarn.application.classpath",
+        System.getProperty("java.class.path"));
     return config;
   }
 
@@ -75,7 +77,8 @@ public void start() throws Exception {
 
     // Start MR server
     LOG.info("Starting MR cluster");
-    mrCluster = new MiniMRCluster(0, 0, 1, dfsCluster.getFileSystem().getUri().toString(), 1, null, null, null, new JobConf(config));
+    mrCluster = new MiniMRCluster(0, 0, 1, dfsCluster.getFileSystem().getUri()
+        .toString(), 1, null, null, null, new JobConf(config));
     LOG.info("Started MR cluster");
     config = prepareConfiguration(mrCluster.createJobConf());
   }

test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java

@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.test.hadoop;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/*
+* This class enables running tests against a real cluster.
+* From the command line, it expects "sqoop.hadoop.config.path"
+* variable to point to directory containing cluster config files,
+* otherwise it tries loading from default location: /etc/hadoop/conf.
+*/
+public class HadoopRealClusterRunner extends HadoopRunner {
+
+  private static final Logger LOG = Logger
+      .getLogger(HadoopRealClusterRunner.class);
+
+  /*
+  * This method loads config files for real cluster.
+  * core-site.xml, mapred-site.xml and hdfs-site.xml are mandatory
+  * while yarn-site.xml is optional to let tests execute against old
+  * (non-yarn based) M/R clusters.
+   */
+  @Override
+  public Configuration prepareConfiguration(Configuration config)
+      throws Exception {
+    String configPath = System.getProperty(
+        "sqoop.hadoop.config.path", "/etc/hadoop/conf");
+    LOG.debug("Config path is: " + configPath);
+
+    File dir = new File(configPath);
+    String [] files = dir.list(new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        return name.endsWith("-site.xml");
+      }
+    });
+
+    if(files == null) {
+      throw new FileNotFoundException("Hadoop config files not found: " + configPath);
+    }
+
+    // Add each config file to configuration object
+    for (String file : files) {
+      LOG.info("Found hadoop configuration file " + file);
+      config.addResource(new Path(configPath, file));
+    }
+    return config;
+  }
+
+  @Override
+  public void start() throws Exception {
+    // Do nothing
+
+  }
+
+  @Override
+  public void stop() throws Exception {
+    // Do nothing
+
+  }
+
+}

test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java

@@ -22,15 +22,17 @@
 
 /**
  * Hadoop cluster runner for testing purpose.
- *
+ * 
- * Runner provides methods for bootstrapping and using Hadoop cluster.
+ * Runner provides methods for bootstrapping and using Hadoop cluster. This
- * This abstract implementation is agnostic about in what mode Hadoop is running.
+ * abstract implementation is agnostic about in what mode Hadoop is running.
- * Each mode will have it's own concrete implementation (for example LocalJobRunner, MiniCluster or Real existing cluster).
+ * Each mode will have it's own concrete implementation (for example
+ * LocalJobRunner, MiniCluster or Real existing cluster).
  */
 public abstract class HadoopRunner {
 
   /**
-   * Temporary path that can be used as a root for other directories storing various data like logs or stored HDFS files.
+   * Temporary path that can be used as a root for other directories storing
+   * various data like logs or stored HDFS files.
    */
   private String temporaryPath;
 
@@ -40,12 +42,14 @@ public abstract class HadoopRunner {
   protected Configuration config = null;
 
   /**
-   * Prepare configuration object.
+   * Prepare configuration object. This method should be called once before the
-   * This method should be called once before the start method is called.
+   * start method is called.
    *
-   * @param config is the configuration object to prepare.
+   * @param config
+   *          is the configuration object to prepare.
    */
-  abstract public Configuration prepareConfiguration(Configuration config);
+  abstract public Configuration prepareConfiguration(Configuration config)
+      throws Exception;
 
   /**
    * Start hadoop cluster.
@@ -89,8 +93,8 @@ public void setTemporaryPath(String temporaryPath) {
   }
 
   /**
-   * Return directory on local filesystem where logs and other
+   * Return directory on local filesystem where logs and other data generated by
-   * data generated by the Hadoop Cluster should be stored.
+   * the Hadoop Cluster should be stored.
    *
    * @return
    */
@@ -99,8 +103,8 @@ public String getDataDir() {
   }
 
   /**
-   * Return directory on local filesystem where logs and other
+   * Return directory on local filesystem where logs and other data generated by
-   * data generated by the Hadoop Cluster should be stored.
+   * the Hadoop Cluster should be stored.
    *
    * @return
    */