From 921d96b621ba97c087babb2c2966394f1ba7aeca Mon Sep 17 00:00:00 2001 From: caoliang <245623257@qq.com> Date: Tue, 6 Dec 2022 18:17:52 +0800 Subject: [PATCH] update selectdbwriter plugin --- selectdbwriter/doc/selectdbwriter.md | 2 +- selectdbwriter/doc/stream2selectdb.json | 2 +- .../writer/selectdbwriter/DelimiterParser.java | 6 +++--- .../selectdbwriter/SelectdbWriterManager.java | 15 +++++++++------ 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/selectdbwriter/doc/selectdbwriter.md b/selectdbwriter/doc/selectdbwriter.md index ce4c1f23..cdf39263 100644 --- a/selectdbwriter/doc/selectdbwriter.md +++ b/selectdbwriter/doc/selectdbwriter.md @@ -349,7 +349,7 @@ c.再次尝试编译。 * **maxBatchRows** - - 描述:每批次导入数据的最大行数。和 **maxBatchSize** 共同控制每批次的导入数量。每批次数据达到两个阈值之一,即开始导入这一批次的数据。 + - 描述:每批次导入数据的最大行数。和 **batchSize** 共同控制每批次的导入数量。每批次数据达到两个阈值之一,即开始导入这一批次的数据。 - 必选:否 - 默认值:500000 diff --git a/selectdbwriter/doc/stream2selectdb.json b/selectdbwriter/doc/stream2selectdb.json index 94038810..d5e14c48 100644 --- a/selectdbwriter/doc/stream2selectdb.json +++ b/selectdbwriter/doc/stream2selectdb.json @@ -75,7 +75,7 @@ } ], "maxBatchRows":200000, - "maxBatchByteSize":53687091200 + "batchSize":53687091200 } } } diff --git a/selectdbwriter/src/main/java/com/alibaba/datax/plugin/writer/selectdbwriter/DelimiterParser.java b/selectdbwriter/src/main/java/com/alibaba/datax/plugin/writer/selectdbwriter/DelimiterParser.java index 10572eae..fa6b397c 100644 --- a/selectdbwriter/src/main/java/com/alibaba/datax/plugin/writer/selectdbwriter/DelimiterParser.java +++ b/selectdbwriter/src/main/java/com/alibaba/datax/plugin/writer/selectdbwriter/DelimiterParser.java @@ -18,14 +18,14 @@ public class DelimiterParser { String hexStr = sp.substring(2); // check hex str if (hexStr.isEmpty()) { - throw new RuntimeException("Failed to parse delimiter: `Hex str is empty`"); + throw new RuntimeException("Failed to parse delimiter: Hex str is empty"); } if (hexStr.length() % 2 != 0) { - throw new RuntimeException("Failed to parse delimiter: `Hex str length error`"); + throw new RuntimeException("Failed to parse delimiter: Hex str length error"); } for (char hexChar : hexStr.toUpperCase().toCharArray()) { if (HEX_STRING.indexOf(hexChar) == -1) { - throw new RuntimeException("Failed to parse delimiter: `Hex str format error`"); + throw new RuntimeException("Failed to parse delimiter: Hex str format error"); } } // transform to separator diff --git a/selectdbwriter/src/main/java/com/alibaba/datax/plugin/writer/selectdbwriter/SelectdbWriterManager.java b/selectdbwriter/src/main/java/com/alibaba/datax/plugin/writer/selectdbwriter/SelectdbWriterManager.java index 479bc5c9..ca04e135 100644 --- a/selectdbwriter/src/main/java/com/alibaba/datax/plugin/writer/selectdbwriter/SelectdbWriterManager.java +++ b/selectdbwriter/src/main/java/com/alibaba/datax/plugin/writer/selectdbwriter/SelectdbWriterManager.java @@ -76,11 +76,13 @@ public class SelectdbWriterManager { batchSize += bts.length; if (batchCount >= options.getBatchRows() || batchSize >= options.getBatchSize()) { String label = createBatchLabel(); - LOG.debug(String.format("buffer Sinking triggered: rows[%d] label [%s].", batchCount, label)); + if(LOG.isDebugEnabled()){ + LOG.debug(String.format("buffer Sinking triggered: rows[%d] label [%s].", batchCount, label)); + } flush(label, false); } } catch (Exception e) { - throw new SelectdbWriterException("Writing records to Doris failed.", e); + throw new SelectdbWriterException("Writing records to selectdb failed.", e); } } @@ -107,13 +109,14 @@ public class SelectdbWriterManager { closed = true; try { String label = createBatchLabel(); - if (batchCount > 0) - LOG.debug(String.format("Selectdb Sink is about to close: label[%s].", label)); + if (batchCount > 0) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Selectdb Sink is about to close: label[%s].", label)); + } + } flush(label, true); } catch (Exception e) { throw new RuntimeException("Writing records to Selectdb failed.", e); - } finally { - this.visitor.close(); } } checkFlushException();