github/DataX
5
0
Fork 0
mirror of https://github.com/alibaba/DataX.git synced 2025-05-02 02:31:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
jiafeng.zhang 2022-10-09 19:41:07 +08:00
parent a4c8d2176e
commit 0efbc5df08
9 changed files with 41 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
doriswriter/doc/doriswriter.md
View File

@ -15,66 +15,50 @@ DorisWriter 通过Doris原生支持Stream load方式导入数据， DorisWriter
```
{
"job": {
"setting": {
"speed": {
"channel": 1
},
"errorLimit": {
"record": 0,
"percentage": 0
}
},
"content": [
{
"reader": {
"name": "streamreader",
"name": "mysqlreader",
"parameter": {
"column": [
"column": ["k1", "k2", "k3"],
"connection": [
{
"value": "皮蛋1",
"type": "string"
},
{
"value": "皮蛋2",
"type": "string"
},
{
"value": "111",
"type": "long"
},
{
"value": "222",
"type": "long"
"jdbcUrl": ["jdbc:mysql://127.0.0.1:3306/db1"],
"table": ["t1"]
}
],
"sliceRecordCount": 100
"username": "root",
"password": "",
"where": ""
}
},
"writer": {
"name": "doriswriter",
"parameter": {
"feLoadUrl": ["127.0.0.1:8030", "127.0.0.2:8030", "127.0.0.3:8030"],
"beLoadUrl": ["192.168.10.1:8040", "192.168.10.2:8040", "192.168.10.3:8040"],
"jdbcUrl": "jdbc:mysql://127.0.0.1:9030/",
"loadUrl": ["127.0.0.1:8030"],
"loadProps": {},
"database": "db1",
"table": "t1",
"column": ["k1", "k2", "v1", "v2"],
"column": ["k1", "k2", "k3"],
"username": "root",
"password": "",
"postSql": [],
"preSql": [],
"loadProps": {
},
"maxBatchRows" : 500000,
"maxBatchByteSize" : 104857600,
"labelPrefix": "my_prefix",
"format":"csv"
}
}
}
"connection": [
"jdbcUrl":"jdbc:mysql://127.0.0.1:9030/demo",
"table":["xxx"],
"selectedDatabase":"xxxx"
]
}
}
}
],
"setting": {
"speed": {
"channel": "1"
}
}
}
}
```
### 3.2 参数说明
@ -91,12 +75,6 @@ DorisWriter 通过Doris原生支持Stream load方式导入数据， DorisWriter
- 必选：是
- 默认值：无
* **beLoadUrl**
- 描述：和 **feLoadUrl** 二选一。作为 Stream Load 的连接目标。格式为 "ip:port"。其中 IP 是 BE 节点 IP，port 是 BE 节点的 webserver_port。可以填写多个，doriswriter 将以轮询的方式访问。
- 必选：否
- 默认值：无
* **username**
- 描述：访问Doris数据库的用户名
@ -140,15 +118,15 @@ DorisWriter 通过Doris原生支持Stream load方式导入数据， DorisWriter
- 默认值：无
* **batchSizeRows**
* **maxBatchRows**
- 描述：每批次导入数据的最大行数。和 **batchByteSize** 共同控制每批次的导入数量。每批次数据达到两个阈值之一，即开始导入这一批次的数据。
- 描述：每批次导入数据的最大行数。和 **maxBatchSize** 共同控制每批次的导入数量。每批次数据达到两个阈值之一，即开始导入这一批次的数据。
- 必选：否
- 默认值：500000
* **batchByteSize**
* **maxBatchSize**
- 描述：每批次导入数据的最大数据量。和 **batchSizeRows** 共同控制每批次的导入数量。每批次数据达到两个阈值之一，即开始导入这一批次的数据。
- 描述：每批次导入数据的最大数据量。和 **maxBatchRows** 共同控制每批次的导入数量。每批次数据达到两个阈值之一，即开始导入这一批次的数据。
- 必选：否
- 默认值：104857600

2
doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisBaseSerializer.java Normal file
View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisBaseSerializer {
}

2
doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCsvSerializer.java Normal file
View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisCsvSerializer {
}

2
doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisDelimiterParser.java Normal file
View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisDelimiterParser {
}

2
doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonSerializer.java Normal file
View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisJsonSerializer {
}

2
doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisSerializer.java Normal file
View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisSerializer {
}

2
doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisSerializerFactory.java Normal file
View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisSerializerFactory {
}

2
doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisStreamLoadExcetion.java Normal file
View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisStreamLoadExcetion {
}

1
doriswriter/src/main/resources/plugin_job_template.json
View File

@ -7,6 +7,7 @@
"preSql": [],
"postSql": [],
"beLoadUrl": [],
"loadUrl": [],
"loadProps": {},
"connection": [
{