diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties index 2895530f..620146d5 100755 --- a/dist/src/main/conf/sqoop.properties +++ b/dist/src/main/conf/sqoop.properties @@ -143,6 +143,9 @@ org.apache.sqoop.submission.engine=org.apache.sqoop.submission.mapreduce.Mapredu # Hadoop configuration directory org.apache.sqoop.submission.engine.mapreduce.configuration.directory=/etc/hadoop/conf/ +# Log level for Sqoop Mapper/Reducer +org.apache.sqoop.submission.engine.mapreduce.configuration.loglevel=INFO + # # Execution engine configuration # diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java index 89681983..6cbdac49 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java @@ -72,6 +72,8 @@ public final class MRJobConstants extends Constants { public static final String JOB_CONNECTOR_TO_NAME = PREFIX_JOB_CONFIG + "connector.to.name"; + public static final String SYSPROP_MR_LOG_LEVEL = "sqoop.mr.loglevel"; + // Hadoop specific constants // We're using constants from Hadoop 1. Hadoop 2 has different names, but // provides backward compatibility layer for those names as well. diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java index 1e1b237b..84103912 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java @@ -268,6 +268,9 @@ private MRConfigurationUtils() { } public static void configureLogging() { + if (System.getProperty(MRJobConstants.SYSPROP_MR_LOG_LEVEL) == null) { + System.setProperty(MRJobConstants.SYSPROP_MR_LOG_LEVEL, "INFO"); + } try (InputStream resourceAsStream = SqoopMapper.class.getResourceAsStream("/META-INF/log4j.properties")) { Properties props = new Properties(); diff --git a/execution/mapreduce/src/main/resources/META-INF/log4j.properties b/execution/mapreduce/src/main/resources/META-INF/log4j.properties index 2a577141..e42c7915 100644 --- a/execution/mapreduce/src/main/resources/META-INF/log4j.properties +++ b/execution/mapreduce/src/main/resources/META-INF/log4j.properties @@ -14,7 +14,7 @@ # limitations under the License. # Set root logger level to INFO and its only appender to A1. -log4j.logger.org.apache.sqoop=INFO, A1 +log4j.logger.org.apache.sqoop=${sqoop.mr.loglevel}, A1 # A1 is set to be a ConsoleAppender. log4j.appender.A1=org.apache.log4j.ConsoleAppender diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java index 93b07255..213da614 100644 --- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java +++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java @@ -27,6 +27,9 @@ public class Constants { public static final String CONF_CONFIG_DIR = PREFIX_MAPREDUCE + "configuration.directory"; + public static final String CONF_SQOOP_MR_LOG_LEVEL = + PREFIX_MAPREDUCE + "configuration.loglevel"; + public static final String SQOOP_JOB = "sqoop.job"; private Constants() { diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java index c03bf39a..0dc4ef06 100644 --- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java +++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java @@ -34,6 +34,7 @@ import org.apache.hadoop.mapred.JobStatus; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.log4j.Logger; import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.MapContext; @@ -107,6 +108,15 @@ public void initialize(MapContext context, String prefix) { } } + // Log level for sqoop mapper and reducer + String mrLogLevel = context.getString(prefix + Constants.CONF_SQOOP_MR_LOG_LEVEL); + if (globalConfiguration.get(MRJobConfig.MR_AM_COMMAND_OPTS) == null) { + globalConfiguration.set(MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS + " -Dsqoop.mr.loglevel=" + mrLogLevel); + } else { + globalConfiguration.set(MRJobConfig.MR_AM_COMMAND_OPTS, globalConfiguration.get(MRJobConfig.MR_AM_COMMAND_OPTS) + " -Dsqoop.mr.loglevel=" + mrLogLevel); + } + + // Save our own property inside the job to easily identify Sqoop jobs globalConfiguration.setBoolean(Constants.SQOOP_JOB, true); diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java index 4ff97e70..c8da7af3 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java @@ -193,6 +193,7 @@ protected Map getSubmissionEngineConfiguration() { properties.put("org.apache.sqoop.submission.engine", "org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine"); properties.put("org.apache.sqoop.submission.engine.mapreduce.configuration.directory", getConfigurationPath()); + properties.put("org.apache.sqoop.submission.engine.mapreduce.configuration.loglevel", "DEBUG"); return properties; }