diff --git a/doriswriter/pom.xml b/doriswriter/pom.xml index 2bbf1d22..1524e358 100755 --- a/doriswriter/pom.xml +++ b/doriswriter/pom.xml @@ -41,6 +41,11 @@ commons-codec 1.9 + + org.apache.commons + commons-lang3 + 3.12.0 + commons-logging commons-logging @@ -108,6 +113,7 @@ + org.apache.commons:commons-lang3 commons-codec:commons-codec commons-logging:* org.apache.httpcomponents:httpclient diff --git a/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisBaseSerializer.java b/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisBaseSerializer.java new file mode 100644 index 00000000..a46fc442 --- /dev/null +++ b/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisBaseSerializer.java @@ -0,0 +1,52 @@ +package com.dorisdb.connector.datax.plugin.writer.doriswriter.row; + +import java.util.TimeZone; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.DateColumn; +import com.alibaba.datax.common.element.Column.Type; +import com.alibaba.datax.common.util.Configuration; + +import org.apache.commons.lang3.time.DateFormatUtils; + +public class DorisBaseSerializer { + + static String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; + + static String dateFormat = "yyyy-MM-dd"; + + static String timeFormat = "HH:mm:ss"; + + static String timeZone = "GMT+8"; + + static TimeZone timeZoner = TimeZone.getTimeZone(DorisBaseSerializer.timeZone); + + static void init(final Configuration configuration) { + DorisBaseSerializer.datetimeFormat = configuration.getString("common.column.datetimeFormat", datetimeFormat); + DorisBaseSerializer.timeFormat = configuration.getString("common.column.timeFormat", timeFormat); + DorisBaseSerializer.dateFormat = configuration.getString("common.column.dateFormat", dateFormat); + DorisBaseSerializer.timeZone = configuration.getString("common.column.timeZone", DorisBaseSerializer.timeZone); + DorisBaseSerializer.timeZoner = TimeZone.getTimeZone(DorisBaseSerializer.timeZone); + } + + protected String fieldConvertion(Column col) { + if (null == col.getRawData()) { + return null; + } + if (Type.DATE != col.getType()) { + return col.asString(); + } + DateColumn.DateType type = ((DateColumn)col).getSubType(); + if (type == DateColumn.DateType.DATE) { + return DateFormatUtils.format(col.asDate(), DorisBaseSerializer.dateFormat, DorisBaseSerializer.timeZoner); + } + if (type == DateColumn.DateType.TIME) { + return DateFormatUtils.format(col.asDate(), DorisBaseSerializer.timeFormat, DorisBaseSerializer.timeZoner); + } + if (type == DateColumn.DateType.DATETIME) { + return DateFormatUtils.format(col.asDate(), DorisBaseSerializer.datetimeFormat, DorisBaseSerializer.timeZoner); + } + return null; + } + +} diff --git a/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisCsvSerializer.java b/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisCsvSerializer.java index 7a22b212..866c8d87 100644 --- a/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisCsvSerializer.java +++ b/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisCsvSerializer.java @@ -6,7 +6,7 @@ import com.alibaba.datax.common.element.Record; import com.google.common.base.Strings; -public class DorisCsvSerializer implements DorisISerializer { +public class DorisCsvSerializer extends DorisBaseSerializer implements DorisISerializer { private static final long serialVersionUID = 1L; @@ -22,7 +22,7 @@ public class DorisCsvSerializer implements DorisISerializer { public String serialize(Record row) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < row.getColumnNumber(); i++) { - Object value = row.getColumn(i).getRawData(); + String value = fieldConvertion(row.getColumn(i)); sb.append(null == value ? "\\N" : value); if (i < row.getColumnNumber() - 1) { sb.append(columnSeparator); diff --git a/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisJsonSerializer.java b/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisJsonSerializer.java index 5da3e9ae..3f74ac58 100644 --- a/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisJsonSerializer.java +++ b/doriswriter/src/main/java/com/dorisdb/connector/datax/plugin/writer/doriswriter/row/DorisJsonSerializer.java @@ -7,7 +7,7 @@ import java.util.Map; import com.alibaba.datax.common.element.Record; import com.alibaba.fastjson.JSON; -public class DorisJsonSerializer implements DorisISerializer { +public class DorisJsonSerializer extends DorisBaseSerializer implements DorisISerializer { private static final long serialVersionUID = 1L; @@ -25,7 +25,8 @@ public class DorisJsonSerializer implements DorisISerializer { Map rowMap = new HashMap<>(fieldNames.size()); int idx = 0; for (String fieldName : fieldNames) { - rowMap.put(fieldName, row.getColumn(idx++).getRawData()); + rowMap.put(fieldName, fieldConvertion(row.getColumn(idx))); + idx++; } return JSON.toJSONString(rowMap); }