From 91ea2787626b3430509d5a6af7af8fe66e420cc9 Mon Sep 17 00:00:00 2001 From: justdba <49357318+justdba@users.noreply.github.com> Date: Thu, 23 Dec 2021 15:24:55 +0800 Subject: [PATCH 1/4] fix: bit type Data truncation for oceanbase bug #1256 --- .../alibaba/datax/plugin/rdbms/writer/CommonRdbmsWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/CommonRdbmsWriter.java b/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/CommonRdbmsWriter.java index 27b88f44..a81f7790 100755 --- a/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/CommonRdbmsWriter.java +++ b/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/CommonRdbmsWriter.java @@ -530,7 +530,7 @@ public class CommonRdbmsWriter { // warn: bit(1) -> Types.BIT 可使用setBoolean // warn: bit(>1) -> Types.VARBINARY 可使用setBytes case Types.BIT: - if (this.dataBaseType == DataBaseType.MySql) { + if (this.dataBaseType == DataBaseType.MySql || this.dataBaseType == DataBaseType.OceanBase) { preparedStatement.setBoolean(columnIndex + 1, column.asBoolean()); } else { preparedStatement.setString(columnIndex + 1, column.asString()); From 515dd00c43311dd435b3add475162dfb49e95bc9 Mon Sep 17 00:00:00 2001 From: justdba <49357318+justdba@users.noreply.github.com> Date: Thu, 23 Dec 2021 15:51:04 +0800 Subject: [PATCH 2/4] fix: datetime(6) Loss of precision in the last 3 digits bug #1258 --- .../datax/common/element/DateColumn.java | 18 ++++++++++++++++-- .../plugin/rdbms/writer/CommonRdbmsWriter.java | 13 ++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java b/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java index 6626a6fb..108b0493 100755 --- a/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java +++ b/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java @@ -61,6 +61,7 @@ public class DateColumn extends Column { * */ public DateColumn(final java.sql.Timestamp ts) { this(ts == null ? null : ts.getTime()); + this.setRawNano(ts == null ? 0 : ts.getNanos()); this.setSubType(DateType.DATETIME); } @@ -86,8 +87,11 @@ public class DateColumn extends Column { if (null == this.getRawData()) { return null; } - - return new Date((Long)this.getRawData()); + java.sql.Timestamp myts = new java.sql.Timestamp((Long) this.getRawData()); + myts.setNanos(this.getRawNano()); + myts.setNanos(this.getRawNano()); + return myts; + // return new Date((Long)this.getRawData()); } @Override @@ -127,4 +131,14 @@ public class DateColumn extends Column { public void setSubType(DateType subType) { this.subType = subType; } + + public int getRawNano(){ + return rawNano; + } + + public void setRawNano(int rawNano){ + this.rawNano = rawNano; + } +} + } \ No newline at end of file diff --git a/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/CommonRdbmsWriter.java b/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/CommonRdbmsWriter.java index a81f7790..f4a69a6c 100755 --- a/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/CommonRdbmsWriter.java +++ b/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/CommonRdbmsWriter.java @@ -501,17 +501,20 @@ public class CommonRdbmsWriter { case Types.TIMESTAMP: java.sql.Timestamp sqlTimestamp = null; + java.sql.Timestamp tsNano = null; try { - utilDate = column.asDate(); + // utilDate = column.asDate(); + tsNano = (java.sql.Timestamp)(column.asDate()); } catch (DataXException e) { throw new SQLException(String.format( "TIMESTAMP 类型转换错误：[%s]", column)); } - if (null != utilDate) { - sqlTimestamp = new java.sql.Timestamp( - utilDate.getTime()); - } + // if (null != utilDate) { + // sqlTimestamp = new java.sql.Timestamp( + // utilDate.getTime()); + // } + sqlTimestamp=tsNano; preparedStatement.setTimestamp(columnIndex + 1, sqlTimestamp); break; From 24e9ec63b8e36744f43f7065d082fe2b9f9aaf97 Mon Sep 17 00:00:00 2001 From: justdba <49357318+justdba@users.noreply.github.com> Date: Thu, 23 Dec 2021 16:04:56 +0800 Subject: [PATCH 3/4] fix: datetime(6) Loss of precision in the last 3 digits bug #1258 --- .../main/java/com/alibaba/datax/common/element/DateColumn.java | 1 - 1 file changed, 1 deletion(-) diff --git a/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java b/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java index 108b0493..08eae0be 100755 --- a/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java +++ b/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java @@ -89,7 +89,6 @@ public class DateColumn extends Column { } java.sql.Timestamp myts = new java.sql.Timestamp((Long) this.getRawData()); myts.setNanos(this.getRawNano()); - myts.setNanos(this.getRawNano()); return myts; // return new Date((Long)this.getRawData()); } From 692a95692d9066efbc13c42ae17a3ca17d5f9451 Mon Sep 17 00:00:00 2001 From: justdba <49357318+justdba@users.noreply.github.com> Date: Tue, 8 Feb 2022 23:13:37 +0800 Subject: [PATCH 4/4] Update SingleTableSplitUtil.java sql plan cause baddly, remove where (AND IS NOT NULL) suffix in select min(pk),max(pk)... --- .../plugin/rdbms/reader/util/SingleTableSplitUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/util/SingleTableSplitUtil.java b/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/util/SingleTableSplitUtil.java index d9846b39..450f545e 100755 --- a/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/util/SingleTableSplitUtil.java +++ b/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/util/SingleTableSplitUtil.java @@ -282,8 +282,8 @@ public class SingleTableSplitUtil { String pkRangeSQL = String.format(minMaxTemplate, splitPK, splitPK, table); if (StringUtils.isNotBlank(where)) { - pkRangeSQL = String.format("%s WHERE (%s AND %s IS NOT NULL)", - pkRangeSQL, where, splitPK); + pkRangeSQL = String.format("%s WHERE (%s)", + pkRangeSQL, where); } return pkRangeSQL; } @@ -387,4 +387,4 @@ public class SingleTableSplitUtil { } return rangeSql; } -} \ No newline at end of file +}