github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-04 01:00:46 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1080: Sqoop2: Investigate Jenkins test failures

Browse Source 
(Venkat Ranganathan via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2013-06-16 06:40:13 -07:00
parent 66a296ead2
commit 05a73ef168
2 changed files with 28 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
View File

@ -22,8 +22,11 @@
import java.sql.Time; import java.sql.Time;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.sql.Types; import java.sql.Types;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.TimeZone;
import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration; import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
@ -106,27 +109,39 @@ protected List<Partition> partitionDateTimeColumn() {
long minDateValue = 0; long minDateValue = 0;
long maxDateValue = 0; long maxDateValue = 0;
SimpleDateFormat sdf = null;
switch(partitionColumnType) { switch(partitionColumnType) {
case Types.DATE: case Types.DATE:
sdf = new SimpleDateFormat("yyyy-MM-dd");
minDateValue = Date.valueOf(partitionMinValue).getTime(); minDateValue = Date.valueOf(partitionMinValue).getTime();
maxDateValue = Date.valueOf(partitionMaxValue).getTime(); maxDateValue = Date.valueOf(partitionMaxValue).getTime();
break; break;
case Types.TIME: case Types.TIME:
sdf = new SimpleDateFormat("HH:mm:ss");
minDateValue = Time.valueOf(partitionMinValue).getTime(); minDateValue = Time.valueOf(partitionMinValue).getTime();
maxDateValue = Time.valueOf(partitionMaxValue).getTime(); maxDateValue = Time.valueOf(partitionMaxValue).getTime();
break; break;
case Types.TIMESTAMP: case Types.TIMESTAMP:
sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
minDateValue = Timestamp.valueOf(partitionMinValue).getTime(); minDateValue = Timestamp.valueOf(partitionMinValue).getTime();
maxDateValue = Timestamp.valueOf(partitionMaxValue).getTime(); maxDateValue = Timestamp.valueOf(partitionMaxValue).getTime();
break; break;
} }
long tzOffset = TimeZone.getDefault().getRawOffset();
minDateValue += tzOffset;
maxDateValue += tzOffset;
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
long interval = (maxDateValue - minDateValue) / numberPartitions; long interval = (maxDateValue - minDateValue) / numberPartitions;
long remainder = (maxDateValue - minDateValue) % numberPartitions; long remainder = (maxDateValue - minDateValue) % numberPartitions;
if (interval == 0) { if (interval == 0) {
numberPartitions = (int)remainder; numberPartitions = (int)remainder;
} }
long lowerBound; long lowerBound;
long upperBound = minDateValue; long upperBound = minDateValue;
@ -146,6 +161,7 @@ protected List<Partition> partitionDateTimeColumn() {
case Types.TIME: case Types.TIME:
objLB = new Time(lowerBound); objLB = new Time(lowerBound);
objUB = new Time(upperBound); objUB = new Time(upperBound);
break; break;
case Types.TIMESTAMP: case Types.TIMESTAMP:
objLB = new Timestamp(lowerBound); objLB = new Timestamp(lowerBound);
@ -155,9 +171,10 @@ protected List<Partition> partitionDateTimeColumn() {
GenericJdbcImportPartition partition = new GenericJdbcImportPartition(); GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
partition.setConditions( partition.setConditions(
constructDateConditions(objLB, objUB, false)); constructDateConditions(sdf, objLB, objUB, false));
partitions.add(partition); partitions.add(partition);
} }
switch(partitionColumnType) { switch(partitionColumnType) {
case Types.DATE: case Types.DATE:
objLB = new Date(upperBound); objLB = new Date(upperBound);
@ -172,9 +189,11 @@ protected List<Partition> partitionDateTimeColumn() {
objUB = new Timestamp(maxDateValue); objUB = new Timestamp(maxDateValue);
break; break;
} }
GenericJdbcImportPartition partition = new GenericJdbcImportPartition(); GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
partition.setConditions( partition.setConditions(
constructDateConditions(objLB, objUB, true)); constructDateConditions(sdf, objLB, objUB, true));
partitions.add(partition); partitions.add(partition);
return partitions; return partitions;
} }
@ -461,16 +480,16 @@ protected String constructConditions(Object value) {
; ;
} }
protected String constructDateConditions( protected String constructDateConditions(SimpleDateFormat sdf,
Object lowerBound, Object upperBound, boolean lastOne) { Object lowerBound, Object upperBound, boolean lastOne) {
StringBuilder conditions = new StringBuilder(); StringBuilder conditions = new StringBuilder();
conditions.append('\'').append(lowerBound.toString()).append('\''); conditions.append('\'').append(sdf.format((java.util.Date)lowerBound)).append('\'');
conditions.append(" <= "); conditions.append(" <= ");
conditions.append(partitionColumnName); conditions.append(partitionColumnName);
conditions.append(" AND "); conditions.append(" AND ");
conditions.append(partitionColumnName); conditions.append(partitionColumnName);
conditions.append(lastOne ? " <= " : " < "); conditions.append(lastOne ? " <= " : " < ");
conditions.append('\'').append(upperBound.toString()).append('\''); conditions.append('\'').append(sdf.format((java.util.Date)upperBound)).append('\'');
return conditions.toString(); return conditions.toString();
} }

6
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
View File

@ -333,9 +333,9 @@ public void testTimestampPartition() throws Exception {
3); 3);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf); List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
verifyResult(partitions, new String[]{ verifyResult(partitions, new String[]{
"'2013-01-01 01:01:01.123' <= TSCOL AND TSCOL < '2013-05-02 13:14:17.634'", "'2013-01-01 01:01:01.123' <= TSCOL AND TSCOL < '2013-05-02 12:14:17.634'",
"'2013-05-02 13:14:17.634' <= TSCOL AND TSCOL < '2013-09-01 00:27:34.144'", "'2013-05-02 12:14:17.634' <= TSCOL AND TSCOL < '2013-08-31 23:27:34.144'",
"'2013-09-01 00:27:34.144' <= TSCOL AND TSCOL <= '2013-12-31 10:40:50.654'", "'2013-08-31 23:27:34.144' <= TSCOL AND TSCOL <= '2013-12-31 10:40:50.654'",
}); });
} }