github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-11 22:41:50 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1308: Database export with the Generic-JDBC-Connector loses rows

Browse Source 
(Kristian Kottke via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2014-05-08 08:05:59 -07:00
parent 8e978c0423
commit dccaf0b258
2 changed files with 31 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java
View File

@ -61,7 +61,7 @@ public void load(LoaderContext context, ConnectionConfiguration connection, Expo
} }
} }
if (numberOfRows != 0) { if (numberOfRows != 0 || numberOfBatches != 0) {
// execute and commit the remaining rows // execute and commit the remaining rows
executor.executeBatch(true); executor.executeBatch(true);
} }

40
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java
View File

@ -17,32 +17,49 @@
*/ */
package org.apache.sqoop.connector.jdbc; package org.apache.sqoop.connector.jdbc;
import java.sql.ResultSet; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import junit.framework.TestCase; import java.sql.ResultSet;
import java.util.Arrays;
import java.util.Collection;
import org.apache.sqoop.common.MutableContext; import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext; import org.apache.sqoop.common.MutableMapContext;
import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration; import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ExportJobConfiguration; import org.apache.sqoop.connector.jdbc.configuration.ExportJobConfiguration;
import org.apache.sqoop.etl.io.DataReader;
import org.apache.sqoop.job.etl.Loader; import org.apache.sqoop.job.etl.Loader;
import org.apache.sqoop.job.etl.LoaderContext; import org.apache.sqoop.job.etl.LoaderContext;
import org.apache.sqoop.etl.io.DataReader; import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
public class TestExportLoader extends TestCase { @RunWith(Parameterized.class)
public class TestExportLoader {
private final String tableName; private final String tableName;
private GenericJdbcExecutor executor; private GenericJdbcExecutor executor;
private static final int START = -50; private static final int START = -50;
private static final int NUMBER_OF_ROWS = 101;
private int numberOfRows;
public TestExportLoader() { @Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {{50}, {100}, {101}, {150}, {200}});
}
public TestExportLoader(int numberOfRows) {
this.numberOfRows = numberOfRows;
tableName = getClass().getSimpleName().toUpperCase(); tableName = getClass().getSimpleName().toUpperCase();
} }
@Override @Before
public void setUp() { public void setUp() {
executor = new GenericJdbcExecutor(GenericJdbcTestConstants.DRIVER, executor = new GenericJdbcExecutor(GenericJdbcTestConstants.DRIVER,
GenericJdbcTestConstants.URL, null, null); GenericJdbcTestConstants.URL, null, null);
@ -51,14 +68,17 @@ public void setUp() {
executor.executeUpdate("CREATE TABLE " executor.executeUpdate("CREATE TABLE "
+ executor.delimitIdentifier(tableName) + executor.delimitIdentifier(tableName)
+ "(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20))"); + "(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20))");
} else {
executor.deleteTableData(tableName);
} }
} }
@Override @After
public void tearDown() { public void tearDown() {
executor.close(); executor.close();
} }
@Test
public void testInsert() throws Exception { public void testInsert() throws Exception {
MutableContext context = new MutableMapContext(); MutableContext context = new MutableMapContext();
@ -86,7 +106,7 @@ public void testInsert() throws Exception {
assertEquals(String.valueOf(index), rs.getObject(3)); assertEquals(String.valueOf(index), rs.getObject(3));
index++; index++;
} }
assertEquals(NUMBER_OF_ROWS, index-START); assertEquals(numberOfRows, index-START);
} }
public class DummyReader extends DataReader { public class DummyReader extends DataReader {
@ -99,7 +119,7 @@ public void setFieldDelimiter(char fieldDelimiter) {
@Override @Override
public Object[] readArrayRecord() { public Object[] readArrayRecord() {
if (index < NUMBER_OF_ROWS) { if (index < numberOfRows) {
Object[] array = new Object[] { Object[] array = new Object[] {
START + index, START + index,
(double) (START + index), (double) (START + index),