github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 22:34:30 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1696: Sqoop2: TestMapReduce.testOutputFormat is failing on Hadoop 1 profile

Browse Source 
(Jarek Jarcec Cecho via Abraham Elmahrek)
This commit is contained in:
Abraham Elmahrek 2014-11-08 23:26:08 -08:00
parent 4ab85d535b
commit 108803b51b
1 changed files with 40 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
View File

@ -23,10 +23,16 @@
import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.OutputFormat; import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.sqoop.job.io.SqoopWritable; import org.apache.sqoop.job.io.SqoopWritable;
import org.apache.sqoop.job.mr.SqoopSplit; import org.apache.sqoop.job.mr.SqoopSplit;
import org.apache.sqoop.utils.ClassUtils;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class JobUtils { public class JobUtils {
@ -44,7 +50,40 @@ public static boolean runJob(Configuration conf,
job.setOutputKeyClass(SqoopWritable.class); job.setOutputKeyClass(SqoopWritable.class);
job.setOutputValueClass(NullWritable.class); job.setOutputValueClass(NullWritable.class);
return job.waitForCompletion(true); boolean ret = job.waitForCompletion(true);
// Hadoop 1.0 (and 0.20) have nasty bug when job committer is not called in LocalJobRuner
if (isHadoop1()) {
callOutputCommitter(job, output);
}
return ret;
}
/**
* Call output format on given job manually.
*/
private static void callOutputCommitter(Job job, Class<? extends OutputFormat<SqoopWritable, NullWritable>> outputFormat) throws IOException, InterruptedException {
OutputCommitter committer = ((OutputFormat)ClassUtils.instantiate(outputFormat)).getOutputCommitter(null);
JobContext jobContext = mock(JobContext.class);
when(jobContext.getConfiguration()).thenReturn(job.getConfiguration());
committer.commitJob(jobContext);
}
/**
* Detect Hadoop 1.0 installation
*
* @return True if and only if this is Hadoop 1 and below
*/
public static boolean isHadoop1() {
String version = org.apache.hadoop.util.VersionInfo.getVersion();
if (version.matches("\\b0\\.20\\..+\\b")
|| version.matches("\\b1\\.\\d\\.\\d")) {
return true;
}
return false;
} }
private JobUtils() { private JobUtils() {