github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-09 19:09:11 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-788 Import sometimes duplicate some data

Browse Source 
(Jarek Jarcec Cecho)
This commit is contained in:
Bilung Lee 2012-12-18 15:52:46 -08:00
parent 7ca9b44688
commit d9465bba21
3 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java
View File

@ -59,8 +59,8 @@ public void run(ImmutableContext context, Object connectionC, Object jobC, Parti
try {
ResultSetMetaData metaData = resultSet.getMetaData();
int column = metaData.getColumnCount();
Object[] array = new Object[column];
while (resultSet.next()) {
Object[] array = new Object[column];
for (int i = 0; i< column; i++) {
array[i] = resultSet.getObject(i+1);
}

1
execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java
View File

@ -84,6 +84,7 @@ private void extractFile(Path file, long start, long length)
boolean hasNext = filereader.next(line);
while (hasNext) {
datawriter.writeCsvRecord(line.toString());
line = new Text();
hasNext = filereader.next(line);
if(filereader.getPosition() >= end && filereader.syncSeen()) {
break;

2
execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java
View File

@ -104,11 +104,11 @@ private void extractFile(Path file, long start, long length)
// one extra line is read in previous split
start += filereader.readLine(new Text(), 0);
}
Text line = new Text();
int size;
LOG.info("Start position: " + String.valueOf(start));
long next = start;
while (next <= end) {
Text line = new Text();
size = filereader.readLine(line, Integer.MAX_VALUE);
if (size == 0) {
break;