From 0efbc5df08ee486c878bde812353ff3e12e245b0 Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Sun, 9 Oct 2022 19:41:07 +0800 Subject: [PATCH] fix --- doriswriter/doc/doriswriter.md | 74 +++++++------------ .../doriswriter/DorisBaseSerializer.java | 2 + .../doriswriter/DorisCsvSerializer.java | 2 + .../doriswriter/DorisDelimiterParser.java | 2 + .../doriswriter/DorisJsonSerializer.java | 2 + .../writer/doriswriter/DorisSerializer.java | 2 + .../doriswriter/DorisSerializerFactory.java | 2 + .../doriswriter/DorisStreamLoadExcetion.java | 2 + .../main/resources/plugin_job_template.json | 1 + 9 files changed, 41 insertions(+), 48 deletions(-) create mode 100644 doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisBaseSerializer.java create mode 100644 doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCsvSerializer.java create mode 100644 doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisDelimiterParser.java create mode 100644 doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonSerializer.java create mode 100644 doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisSerializer.java create mode 100644 doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisSerializerFactory.java create mode 100644 doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisStreamLoadExcetion.java diff --git a/doriswriter/doc/doriswriter.md b/doriswriter/doc/doriswriter.md index 563de6d8..344cd943 100644 --- a/doriswriter/doc/doriswriter.md +++ b/doriswriter/doc/doriswriter.md @@ -15,64 +15,48 @@ DorisWriter 通过Doris原生支持Stream load方式导入数据， DorisWriter ``` { "job": { - "setting": { - "speed": { - "channel": 1 - }, - "errorLimit": { - "record": 0, - "percentage": 0 - } - }, "content": [ { "reader": { - "name": "streamreader", + "name": "mysqlreader", "parameter": { - "column": [ + "column": ["k1", "k2", "k3"], + "connection": [ { - "value": "皮蛋1", - "type": "string" - }, - { - "value": "皮蛋2", - "type": "string" - }, - { - "value": "111", - "type": "long" - }, - { - "value": "222", - "type": "long" + "jdbcUrl": ["jdbc:mysql://127.0.0.1:3306/db1"], + "table": ["t1"] } ], - "sliceRecordCount": 100 + "username": "root", + "password": "", + "where": "" } }, "writer": { "name": "doriswriter", "parameter": { - "feLoadUrl": ["127.0.0.1:8030", "127.0.0.2:8030", "127.0.0.3:8030"], - "beLoadUrl": ["192.168.10.1:8040", "192.168.10.2:8040", "192.168.10.3:8040"], - "jdbcUrl": "jdbc:mysql://127.0.0.1:9030/", + "loadUrl": ["127.0.0.1:8030"], + "loadProps": {}, "database": "db1", - "table": "t1", - "column": ["k1", "k2", "v1", "v2"], + "column": ["k1", "k2", "k3"], "username": "root", "password": "", "postSql": [], "preSql": [], - "loadProps": { - }, - "maxBatchRows" : 500000, - "maxBatchByteSize" : 104857600, - "labelPrefix": "my_prefix", - "format":"csv" + "connection": [ + "jdbcUrl":"jdbc:mysql://127.0.0.1:9030/demo", + "table":["xxx"], + "selectedDatabase":"xxxx" + ] } } } - ] + ], + "setting": { + "speed": { + "channel": "1" + } + } } } ``` @@ -91,12 +75,6 @@ DorisWriter 通过Doris原生支持Stream load方式导入数据， DorisWriter - 必选：是 - 默认值：无 -* **beLoadUrl** - - - 描述：和 **feLoadUrl** 二选一。作为 Stream Load 的连接目标。格式为 "ip:port"。其中 IP 是 BE 节点 IP，port 是 BE 节点的 webserver_port。可以填写多个，doriswriter 将以轮询的方式访问。 - - 必选：否 - - 默认值：无 - * **username** - 描述：访问Doris数据库的用户名 @@ -140,15 +118,15 @@ DorisWriter 通过Doris原生支持Stream load方式导入数据， DorisWriter - 默认值：无 -* **batchSizeRows** +* **maxBatchRows** - - 描述：每批次导入数据的最大行数。和 **batchByteSize** 共同控制每批次的导入数量。每批次数据达到两个阈值之一，即开始导入这一批次的数据。 + - 描述：每批次导入数据的最大行数。和 **maxBatchSize** 共同控制每批次的导入数量。每批次数据达到两个阈值之一，即开始导入这一批次的数据。 - 必选：否 - 默认值：500000 -* **batchByteSize** +* **maxBatchSize** - - 描述：每批次导入数据的最大数据量。和 **batchSizeRows** 共同控制每批次的导入数量。每批次数据达到两个阈值之一，即开始导入这一批次的数据。 + - 描述：每批次导入数据的最大数据量。和 **maxBatchRows** 共同控制每批次的导入数量。每批次数据达到两个阈值之一，即开始导入这一批次的数据。 - 必选：否 - 默认值：104857600 diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisBaseSerializer.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisBaseSerializer.java new file mode 100644 index 00000000..6179ec71 --- /dev/null +++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisBaseSerializer.java @@ -0,0 +1,2 @@ +package com.alibaba.datax.plugin.writer.doriswriter;public class DorisBaseSerializer { +} diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCsvSerializer.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCsvSerializer.java new file mode 100644 index 00000000..298d1142 --- /dev/null +++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCsvSerializer.java @@ -0,0 +1,2 @@ +package com.alibaba.datax.plugin.writer.doriswriter;public class DorisCsvSerializer { +} diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisDelimiterParser.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisDelimiterParser.java new file mode 100644 index 00000000..e3f4b46c --- /dev/null +++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisDelimiterParser.java @@ -0,0 +1,2 @@ +package com.alibaba.datax.plugin.writer.doriswriter;public class DorisDelimiterParser { +} diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonSerializer.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonSerializer.java new file mode 100644 index 00000000..e747c996 --- /dev/null +++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonSerializer.java @@ -0,0 +1,2 @@ +package com.alibaba.datax.plugin.writer.doriswriter;public class DorisJsonSerializer { +} diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisSerializer.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisSerializer.java new file mode 100644 index 00000000..a974c5e5 --- /dev/null +++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisSerializer.java @@ -0,0 +1,2 @@ +package com.alibaba.datax.plugin.writer.doriswriter;public class DorisSerializer { +} diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisSerializerFactory.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisSerializerFactory.java new file mode 100644 index 00000000..2bb1a1c1 --- /dev/null +++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisSerializerFactory.java @@ -0,0 +1,2 @@ +package com.alibaba.datax.plugin.writer.doriswriter;public class DorisSerializerFactory { +} diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisStreamLoadExcetion.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisStreamLoadExcetion.java new file mode 100644 index 00000000..c8e27e00 --- /dev/null +++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisStreamLoadExcetion.java @@ -0,0 +1,2 @@ +package com.alibaba.datax.plugin.writer.doriswriter;public class DorisStreamLoadExcetion { +} diff --git a/doriswriter/src/main/resources/plugin_job_template.json b/doriswriter/src/main/resources/plugin_job_template.json index 897fe49b..0187e539 100644 --- a/doriswriter/src/main/resources/plugin_job_template.json +++ b/doriswriter/src/main/resources/plugin_job_template.json @@ -7,6 +7,7 @@ "preSql": [], "postSql": [], "beLoadUrl": [], + "loadUrl": [], "loadProps": {}, "connection": [ {