From 75ae6e3ea0931f7f677d3e0bfd5109c8fe85c369 Mon Sep 17 00:00:00 2001 From: Kate Ting Date: Sun, 14 Apr 2013 19:08:34 -0400 Subject: [PATCH] SQOOP-986: Sqoop2: Add detection of local mode to mapreduce submission engine (Jarek Jarcec Cecho via Kate Ting) --- .../mapreduce/MapreduceSubmissionEngine.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java index 001fb02c..dd7b9e77 100644 --- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java +++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java @@ -114,6 +114,10 @@ public boolean accept(File dir, String name) { } catch (IOException e) { throw new SqoopException(MapreduceSubmissionError.MAPREDUCE_0002, e); } + + if(isLocal()) { + LOG.info("Detected MapReduce local mode, some methods might not work correctly."); + } } /** @@ -249,7 +253,14 @@ public boolean submit(SubmissionRequest generalRequest) { job.setOutputKeyClass(request.getOutputKeyClass()); job.setOutputValueClass(request.getOutputValueClass()); - job.submit(); + // If we're in local mode than wait on completion. Local job runner do not + // seems to be exposing API to get previously submitted job which makes + // other methods of the submission engine quite useless. + if(isLocal()) { + job.waitForCompletion(true); + } else { + job.submit(); + } String jobId = job.getJobID().toString(); request.getSummary().setExternalId(jobId); @@ -400,4 +411,13 @@ private Counters convertMapreduceCounters(org.apache.hadoop.mapred.Counters hado return sqoopCounters; } + /** + * Detect MapReduce local mode. + * + * @return True if we're running in local mode + */ + private boolean isLocal() { + return "local".equals(globalConfiguration.get("mapreduce.jobtracker.address")) + || "local".equals(globalConfiguration.get("mapred.job.tracker")); + } }