diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java index 44465b44..140d0f90 100644 --- a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java +++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java @@ -21,14 +21,15 @@ import org.apache.sqoop.test.utils.HdfsUtils; /** - * Represents a local cluster. - * It uses an unchanged Configuration object. - * HadoopRunner implementation that is using LocalJobRunner for executing mapreduce jobs and local filesystem instead of HDFS. + * Represents a local cluster. It uses an unchanged Configuration object. + * HadoopRunner implementation that is using LocalJobRunner for executing + * mapreduce jobs and local filesystem instead of HDFS. */ public class HadoopLocalRunner extends HadoopRunner { @Override - public Configuration prepareConfiguration(Configuration conf) { + public Configuration prepareConfiguration(Configuration conf) + throws Exception { return conf; } diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java index b06dcab5..2c0c4e69 100644 --- a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java +++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java @@ -24,13 +24,13 @@ import org.apache.log4j.Logger; /** - * Represents a minicluster setup. - * It creates a configuration object and mutates it. - * Clients that need to connect to the miniclusters should use - * the provided configuration object. + * Represents a minicluster setup. It creates a configuration object and mutates + * it. Clients that need to connect to the miniclusters should use the provided + * configuration object. */ public class HadoopMiniClusterRunner extends HadoopRunner { - private static final Logger LOG = Logger.getLogger(HadoopMiniClusterRunner.class); + private static final Logger LOG = Logger + .getLogger(HadoopMiniClusterRunner.class); /** * Hadoop HDFS cluster @@ -43,7 +43,8 @@ public class HadoopMiniClusterRunner extends HadoopRunner { protected MiniMRCluster mrCluster; @Override - public Configuration prepareConfiguration(Configuration config) { + public Configuration prepareConfiguration(Configuration config) + throws Exception { config.set("dfs.block.access.token.enable", "false"); config.set("dfs.permissions", "true"); config.set("hadoop.security.authentication", "simple"); @@ -51,7 +52,8 @@ public Configuration prepareConfiguration(Configuration config) { config.set("mapred.tasktracker.reduce.tasks.maximum", "1"); config.set("mapred.submit.replication", "1"); config.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); - config.set("yarn.application.classpath", System.getProperty("java.class.path")); + config.set("yarn.application.classpath", + System.getProperty("java.class.path")); return config; } @@ -75,7 +77,8 @@ public void start() throws Exception { // Start MR server LOG.info("Starting MR cluster"); - mrCluster = new MiniMRCluster(0, 0, 1, dfsCluster.getFileSystem().getUri().toString(), 1, null, null, null, new JobConf(config)); + mrCluster = new MiniMRCluster(0, 0, 1, dfsCluster.getFileSystem().getUri() + .toString(), 1, null, null, null, new JobConf(config)); LOG.info("Started MR cluster"); config = prepareConfiguration(mrCluster.createJobConf()); } diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java new file mode 100644 index 00000000..eabdf0b3 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.test.hadoop; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FilenameFilter; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.log4j.Logger; + +/* +* This class enables running tests against a real cluster. +* From the command line, it expects "sqoop.hadoop.config.path" +* variable to point to directory containing cluster config files, +* otherwise it tries loading from default location: /etc/hadoop/conf. +*/ +public class HadoopRealClusterRunner extends HadoopRunner { + + private static final Logger LOG = Logger + .getLogger(HadoopRealClusterRunner.class); + + /* + * This method loads config files for real cluster. + * core-site.xml, mapred-site.xml and hdfs-site.xml are mandatory + * while yarn-site.xml is optional to let tests execute against old + * (non-yarn based) M/R clusters. + */ + @Override + public Configuration prepareConfiguration(Configuration config) + throws Exception { + String configPath = System.getProperty( + "sqoop.hadoop.config.path", "/etc/hadoop/conf"); + LOG.debug("Config path is: " + configPath); + + File dir = new File(configPath); + String [] files = dir.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith("-site.xml"); + } + }); + + if(files == null) { + throw new FileNotFoundException("Hadoop config files not found: " + configPath); + } + + // Add each config file to configuration object + for (String file : files) { + LOG.info("Found hadoop configuration file " + file); + config.addResource(new Path(configPath, file)); + } + return config; + } + + @Override + public void start() throws Exception { + // Do nothing + + } + + @Override + public void stop() throws Exception { + // Do nothing + + } + +} diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java index 2516ff18..cb4e3842 100644 --- a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java +++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java @@ -22,15 +22,17 @@ /** * Hadoop cluster runner for testing purpose. - * - * Runner provides methods for bootstrapping and using Hadoop cluster. - * This abstract implementation is agnostic about in what mode Hadoop is running. - * Each mode will have it's own concrete implementation (for example LocalJobRunner, MiniCluster or Real existing cluster). + * + * Runner provides methods for bootstrapping and using Hadoop cluster. This + * abstract implementation is agnostic about in what mode Hadoop is running. + * Each mode will have it's own concrete implementation (for example + * LocalJobRunner, MiniCluster or Real existing cluster). */ public abstract class HadoopRunner { /** - * Temporary path that can be used as a root for other directories storing various data like logs or stored HDFS files. + * Temporary path that can be used as a root for other directories storing + * various data like logs or stored HDFS files. */ private String temporaryPath; @@ -40,12 +42,14 @@ public abstract class HadoopRunner { protected Configuration config = null; /** - * Prepare configuration object. - * This method should be called once before the start method is called. + * Prepare configuration object. This method should be called once before the + * start method is called. * - * @param config is the configuration object to prepare. + * @param config + * is the configuration object to prepare. */ - abstract public Configuration prepareConfiguration(Configuration config); + abstract public Configuration prepareConfiguration(Configuration config) + throws Exception; /** * Start hadoop cluster. @@ -89,8 +93,8 @@ public void setTemporaryPath(String temporaryPath) { } /** - * Return directory on local filesystem where logs and other - * data generated by the Hadoop Cluster should be stored. + * Return directory on local filesystem where logs and other data generated by + * the Hadoop Cluster should be stored. * * @return */ @@ -99,8 +103,8 @@ public String getDataDir() { } /** - * Return directory on local filesystem where logs and other - * data generated by the Hadoop Cluster should be stored. + * Return directory on local filesystem where logs and other data generated by + * the Hadoop Cluster should be stored. * * @return */