From 1a48afdac79d3bb05eaf0477b4ce3c87bd4a6493 Mon Sep 17 00:00:00 2001 From: LuckyPickleZZ Date: Thu, 7 Dec 2023 12:02:54 +0800 Subject: [PATCH 1/3] set null value to `NULL` in SqlWriter --- .../unstructuredstorage/writer/SqlWriter.java | 16 ++++++++++++---- .../writer/UnstructuredStorageWriterUtil.java | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java b/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java index 4d6ff713..49cebfd0 100644 --- a/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java +++ b/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java @@ -15,13 +15,15 @@ public class SqlWriter implements UnstructuredWriter { private String quoteChar; private String lineSeparator; private String tableName; + private String nullFormat; private StringBuilder insertPrefix; - public SqlWriter(Writer writer, String quoteChar, String tableName, String lineSeparator, List columnNames) { + public SqlWriter(Writer writer, String quoteChar, String tableName, String lineSeparator, List columnNames, String nullFormat) { this.sqlWriter = writer; this.quoteChar = quoteChar; this.lineSeparator = lineSeparator; this.tableName = tableName; + this.nullFormat = nullFormat; buildInsertPrefix(columnNames); } @@ -33,7 +35,12 @@ public class SqlWriter implements UnstructuredWriter { } StringBuilder sqlPatten = new StringBuilder(4096).append(insertPrefix); - sqlPatten.append(splitedRows.stream().map(e -> "'" + DataXCsvWriter.replace(e, "'", "''") + "'").collect(Collectors.joining(","))); + sqlPatten.append(splitedRows.stream().map(e -> { + if (nullFormat.equals(e)) { + return "NULL"; + } + return "'" + DataXCsvWriter.replace(e, "'", "''") + "'"; + }).collect(Collectors.joining(","))); sqlPatten.append(");").append(lineSeparator); this.sqlWriter.write(sqlPatten.toString()); } @@ -48,9 +55,10 @@ public class SqlWriter implements UnstructuredWriter { sb.append(quoteChar).append(columnName).append(quoteChar); } - int capacity = 16 + tableName.length() + sb.length(); + int capacity = 18 + tableName.length() + sb.length(); this.insertPrefix = new StringBuilder(capacity); - this.insertPrefix.append("INSERT INTO ").append(tableName).append(" (").append(sb).append(")").append(" VALUES("); + this.insertPrefix + .append("INSERT INTO ").append("`").append(tableName).append("`").append(" (").append(sb).append(")").append(" VALUES("); } public void appendCommit() throws IOException { diff --git a/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/UnstructuredStorageWriterUtil.java b/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/UnstructuredStorageWriterUtil.java index 4ce6461c..e74e5698 100755 --- a/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/UnstructuredStorageWriterUtil.java +++ b/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/UnstructuredStorageWriterUtil.java @@ -283,7 +283,8 @@ public class UnstructuredStorageWriterUtil { String lineSeparator = config.getString(Key.LINE_DELIMITER, IOUtils.LINE_SEPARATOR); List headers = config.getList(Key.HEADER, String.class); Preconditions.checkArgument(CollectionUtils.isNotEmpty(headers), "column names are empty"); - unstructuredWriter = new SqlWriter(writer, quoteChar, tableName, lineSeparator, headers); + String nullFormat = config.getString(Key.NULL_FORMAT, Constant.DEFAULT_NULL_FORMAT); + unstructuredWriter = new SqlWriter(writer, quoteChar, tableName, lineSeparator, headers, nullFormat); } return unstructuredWriter; From abf3ea02305afcd40dca235ce44a9e533b8e404f Mon Sep 17 00:00:00 2001 From: LuckyPickleZZ Date: Fri, 8 Dec 2023 11:25:01 +0800 Subject: [PATCH 2/3] fix an error --- .../datax/plugin/unstructuredstorage/writer/SqlWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java b/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java index 49cebfd0..5ab1a5d2 100644 --- a/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java +++ b/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java @@ -58,7 +58,7 @@ public class SqlWriter implements UnstructuredWriter { int capacity = 18 + tableName.length() + sb.length(); this.insertPrefix = new StringBuilder(capacity); this.insertPrefix - .append("INSERT INTO ").append("`").append(tableName).append("`").append(" (").append(sb).append(")").append(" VALUES("); + .append("INSERT INTO ").append(quoteChar).append(tableName).append(quoteChar).append(" (").append(sb).append(")").append(" VALUES("); } public void appendCommit() throws IOException { From 7097651cd80cfabb414781cdf41e4e8029afe4a1 Mon Sep 17 00:00:00 2001 From: LuckyPickleZZ Date: Fri, 8 Dec 2023 11:35:07 +0800 Subject: [PATCH 3/3] fix an error --- .../datax/plugin/unstructuredstorage/writer/SqlWriter.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java b/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java index 5ab1a5d2..18a9c1be 100644 --- a/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java +++ b/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/SqlWriter.java @@ -22,7 +22,7 @@ public class SqlWriter implements UnstructuredWriter { this.sqlWriter = writer; this.quoteChar = quoteChar; this.lineSeparator = lineSeparator; - this.tableName = tableName; + this.tableName = quoteChar + tableName + quoteChar; this.nullFormat = nullFormat; buildInsertPrefix(columnNames); } @@ -55,10 +55,9 @@ public class SqlWriter implements UnstructuredWriter { sb.append(quoteChar).append(columnName).append(quoteChar); } - int capacity = 18 + tableName.length() + sb.length(); + int capacity = 16 + tableName.length() + sb.length(); this.insertPrefix = new StringBuilder(capacity); - this.insertPrefix - .append("INSERT INTO ").append(quoteChar).append(tableName).append(quoteChar).append(" (").append(sb).append(")").append(" VALUES("); + this.insertPrefix.append("INSERT INTO ").append(tableName).append(" (").append(sb).append(")").append(" VALUES("); } public void appendCommit() throws IOException {