From 05a73ef168f65fea155befb9dbbac01b61febed5 Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Sun, 16 Jun 2013 06:40:13 -0700 Subject: [PATCH] SQOOP-1080: Sqoop2: Investigate Jenkins test failures (Venkat Ranganathan via Jarek Jarcec Cecho) --- .../jdbc/GenericJdbcImportPartitioner.java | 31 +++++++++++++++---- .../connector/jdbc/TestImportPartitioner.java | 6 ++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java index eef18f2e..0bb979e9 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java @@ -22,8 +22,11 @@ import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.LinkedList; import java.util.List; +import java.util.TimeZone; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration; @@ -106,27 +109,39 @@ protected List partitionDateTimeColumn() { long minDateValue = 0; long maxDateValue = 0; - + SimpleDateFormat sdf = null; switch(partitionColumnType) { case Types.DATE: + sdf = new SimpleDateFormat("yyyy-MM-dd"); minDateValue = Date.valueOf(partitionMinValue).getTime(); maxDateValue = Date.valueOf(partitionMaxValue).getTime(); break; case Types.TIME: + sdf = new SimpleDateFormat("HH:mm:ss"); minDateValue = Time.valueOf(partitionMinValue).getTime(); maxDateValue = Time.valueOf(partitionMaxValue).getTime(); break; case Types.TIMESTAMP: + sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); minDateValue = Timestamp.valueOf(partitionMinValue).getTime(); maxDateValue = Timestamp.valueOf(partitionMaxValue).getTime(); break; } + + long tzOffset = TimeZone.getDefault().getRawOffset(); + + minDateValue += tzOffset; + maxDateValue += tzOffset; + + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + long interval = (maxDateValue - minDateValue) / numberPartitions; long remainder = (maxDateValue - minDateValue) % numberPartitions; if (interval == 0) { numberPartitions = (int)remainder; } + long lowerBound; long upperBound = minDateValue; @@ -146,6 +161,7 @@ protected List partitionDateTimeColumn() { case Types.TIME: objLB = new Time(lowerBound); objUB = new Time(upperBound); + break; case Types.TIMESTAMP: objLB = new Timestamp(lowerBound); @@ -155,9 +171,10 @@ protected List partitionDateTimeColumn() { GenericJdbcImportPartition partition = new GenericJdbcImportPartition(); partition.setConditions( - constructDateConditions(objLB, objUB, false)); + constructDateConditions(sdf, objLB, objUB, false)); partitions.add(partition); } + switch(partitionColumnType) { case Types.DATE: objLB = new Date(upperBound); @@ -172,9 +189,11 @@ protected List partitionDateTimeColumn() { objUB = new Timestamp(maxDateValue); break; } + + GenericJdbcImportPartition partition = new GenericJdbcImportPartition(); partition.setConditions( - constructDateConditions(objLB, objUB, true)); + constructDateConditions(sdf, objLB, objUB, true)); partitions.add(partition); return partitions; } @@ -461,16 +480,16 @@ protected String constructConditions(Object value) { ; } - protected String constructDateConditions( + protected String constructDateConditions(SimpleDateFormat sdf, Object lowerBound, Object upperBound, boolean lastOne) { StringBuilder conditions = new StringBuilder(); - conditions.append('\'').append(lowerBound.toString()).append('\''); + conditions.append('\'').append(sdf.format((java.util.Date)lowerBound)).append('\''); conditions.append(" <= "); conditions.append(partitionColumnName); conditions.append(" AND "); conditions.append(partitionColumnName); conditions.append(lastOne ? " <= " : " < "); - conditions.append('\'').append(upperBound.toString()).append('\''); + conditions.append('\'').append(sdf.format((java.util.Date)upperBound)).append('\''); return conditions.toString(); } diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java index 522a5159..1a3601a2 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java @@ -333,9 +333,9 @@ public void testTimestampPartition() throws Exception { 3); List partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf); verifyResult(partitions, new String[]{ - "'2013-01-01 01:01:01.123' <= TSCOL AND TSCOL < '2013-05-02 13:14:17.634'", - "'2013-05-02 13:14:17.634' <= TSCOL AND TSCOL < '2013-09-01 00:27:34.144'", - "'2013-09-01 00:27:34.144' <= TSCOL AND TSCOL <= '2013-12-31 10:40:50.654'", + "'2013-01-01 01:01:01.123' <= TSCOL AND TSCOL < '2013-05-02 12:14:17.634'", + "'2013-05-02 12:14:17.634' <= TSCOL AND TSCOL < '2013-08-31 23:27:34.144'", + "'2013-08-31 23:27:34.144' <= TSCOL AND TSCOL <= '2013-12-31 10:40:50.654'", }); }