github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-10 02:40:49 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-788 Import sometimes duplicate some data

Browse Source 
(Jarek Jarcec Cecho)
This commit is contained in:
Bilung Lee 2012-12-18 15:52:46 -08:00
parent 7ca9b44688
commit d9465bba21
3 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java
View File

@ -59,8 +59,8 @@ public void run(ImmutableContext context, Object connectionC, Object jobC, Parti
try { try {
ResultSetMetaData metaData = resultSet.getMetaData(); ResultSetMetaData metaData = resultSet.getMetaData();
int column = metaData.getColumnCount(); int column = metaData.getColumnCount();
Object[] array = new Object[column];
while (resultSet.next()) { while (resultSet.next()) {
Object[] array = new Object[column];
for (int i = 0; i< column; i++) { for (int i = 0; i< column; i++) {
array[i] = resultSet.getObject(i+1); array[i] = resultSet.getObject(i+1);
} }

1
execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java
View File

@ -84,6 +84,7 @@ private void extractFile(Path file, long start, long length)
boolean hasNext = filereader.next(line); boolean hasNext = filereader.next(line);
while (hasNext) { while (hasNext) {
datawriter.writeCsvRecord(line.toString()); datawriter.writeCsvRecord(line.toString());
line = new Text();
hasNext = filereader.next(line); hasNext = filereader.next(line);
if(filereader.getPosition() >= end && filereader.syncSeen()) { if(filereader.getPosition() >= end && filereader.syncSeen()) {
break; break;

2
execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java
View File

@ -104,11 +104,11 @@ private void extractFile(Path file, long start, long length)
// one extra line is read in previous split // one extra line is read in previous split
start += filereader.readLine(new Text(), 0); start += filereader.readLine(new Text(), 0);
} }
Text line = new Text();
int size; int size;
LOG.info("Start position: " + String.valueOf(start)); LOG.info("Start position: " + String.valueOf(start));
long next = start; long next = start;
while (next <= end) { while (next <= end) {
Text line = new Text();
size = filereader.readLine(line, Integer.MAX_VALUE); size = filereader.readLine(line, Integer.MAX_VALUE);
if (size == 0) { if (size == 0) {
break; break;