github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-08 21:12:12 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-720: Improve error handling when exporting malformed text data

Browse Source 
(Jarek Jarcec Cecho via Cheolsoo Park)
This commit is contained in:
Cheolsoo Park 2012-11-30 16:13:44 -08:00
parent a840f41fc7
commit 528f7a8bb5
1 changed files with 36 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/java/org/apache/sqoop/mapreduce/TextExportMapper.java
View File

@ -23,10 +23,13 @@
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.util.ReflectionUtils;
import com.cloudera.sqoop.lib.RecordParser;
import com.cloudera.sqoop.lib.SqoopRecord;
import com.cloudera.sqoop.mapreduce.AutoProgressMapper;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* Converts an input record from a string representation to a parsed Sqoop
@ -36,6 +39,9 @@
public class TextExportMapper
extends AutoProgressMapper<LongWritable, Text, SqoopRecord, NullWritable> {
public static final Log LOG =
LogFactory.getLog(TextExportMapper.class.getName());
private SqoopRecord recordImpl;
public TextExportMapper() {
@ -76,8 +82,35 @@ public void map(LongWritable key, Text val, Context context)
try {
recordImpl.parse(val);
context.write(recordImpl, NullWritable.get());
} catch (RecordParser.ParseError pe) {
throw new IOException("Could not parse record: " + val, pe);
} catch (Exception e) {
// Something bad has happened
LOG.error("");
LOG.error("Exception raised during data export");
LOG.error("");
LOG.error("Exception: ", e);
LOG.error("On input: " + val);
InputSplit is = context.getInputSplit();
if (is instanceof FileSplit) {
LOG.error("On input file: " + ((FileSplit)is).getPath());
} else if (is instanceof CombineFileSplit) {
LOG.error("On input file: "
+ context.getConfiguration().get("map.input.file"));
}
LOG.error("At position " + key);
LOG.error("");
LOG.error("Currently processing split:");
LOG.error(is);
LOG.error("");
LOG.error("This issue might not necessarily be caused by current input");
LOG.error("due to the batching nature of export.");
LOG.error("");
throw new IOException("Can't export data, please check task tracker logs",
e);
}
}
}