This commit is contained in:
jiafeng.zhang 2022-10-09 19:41:07 +08:00
parent a4c8d2176e
commit 0efbc5df08
9 changed files with 41 additions and 48 deletions

View File

@ -15,65 +15,49 @@ DorisWriter 通过Doris原生支持Stream load方式导入数据 DorisWriter
``` ```
{ {
"job": { "job": {
"setting": {
"speed": {
"channel": 1
},
"errorLimit": {
"record": 0,
"percentage": 0
}
},
"content": [ "content": [
{ {
"reader": { "reader": {
"name": "streamreader", "name": "mysqlreader",
"parameter": { "parameter": {
"column": [ "column": ["k1", "k2", "k3"],
"connection": [
{ {
"value": "皮蛋1", "jdbcUrl": ["jdbc:mysql://127.0.0.1:3306/db1"],
"type": "string" "table": ["t1"]
},
{
"value": "皮蛋2",
"type": "string"
},
{
"value": "111",
"type": "long"
},
{
"value": "222",
"type": "long"
} }
], ],
"sliceRecordCount": 100 "username": "root",
"password": "",
"where": ""
} }
}, },
"writer": { "writer": {
"name": "doriswriter", "name": "doriswriter",
"parameter": { "parameter": {
"feLoadUrl": ["127.0.0.1:8030", "127.0.0.2:8030", "127.0.0.3:8030"], "loadUrl": ["127.0.0.1:8030"],
"beLoadUrl": ["192.168.10.1:8040", "192.168.10.2:8040", "192.168.10.3:8040"], "loadProps": {},
"jdbcUrl": "jdbc:mysql://127.0.0.1:9030/",
"database": "db1", "database": "db1",
"table": "t1", "column": ["k1", "k2", "k3"],
"column": ["k1", "k2", "v1", "v2"],
"username": "root", "username": "root",
"password": "", "password": "",
"postSql": [], "postSql": [],
"preSql": [], "preSql": [],
"loadProps": { "connection": [
}, "jdbcUrl":"jdbc:mysql://127.0.0.1:9030/demo",
"maxBatchRows" : 500000, "table":["xxx"],
"maxBatchByteSize" : 104857600, "selectedDatabase":"xxxx"
"labelPrefix": "my_prefix",
"format":"csv"
}
}
}
] ]
} }
}
}
],
"setting": {
"speed": {
"channel": "1"
}
}
}
} }
``` ```
@ -91,12 +75,6 @@ DorisWriter 通过Doris原生支持Stream load方式导入数据 DorisWriter
- 必选:是 - 必选:是
- 默认值:无 - 默认值:无
* **beLoadUrl**
- 描述:和 **feLoadUrl** 二选一。作为 Stream Load 的连接目标。格式为 "ip:port"。其中 IP 是 BE 节点 IPport 是 BE 节点的 webserver_port。可以填写多个doriswriter 将以轮询的方式访问。
- 必选:否
- 默认值:无
* **username** * **username**
- 描述访问Doris数据库的用户名 - 描述访问Doris数据库的用户名
@ -140,15 +118,15 @@ DorisWriter 通过Doris原生支持Stream load方式导入数据 DorisWriter
- 默认值:无 - 默认值:无
* **batchSizeRows** * **maxBatchRows**
- 描述:每批次导入数据的最大行数。和 **batchByteSize** 共同控制每批次的导入数量。每批次数据达到两个阈值之一,即开始导入这一批次的数据。 - 描述:每批次导入数据的最大行数。和 **maxBatchSize** 共同控制每批次的导入数量。每批次数据达到两个阈值之一,即开始导入这一批次的数据。
- 必选:否 - 必选:否
- 默认值500000 - 默认值500000
* **batchByteSize** * **maxBatchSize**
- 描述:每批次导入数据的最大数据量。和 **batchSizeRows** 共同控制每批次的导入数量。每批次数据达到两个阈值之一,即开始导入这一批次的数据。 - 描述:每批次导入数据的最大数据量。和 **maxBatchRows** 共同控制每批次的导入数量。每批次数据达到两个阈值之一,即开始导入这一批次的数据。
- 必选:否 - 必选:否
- 默认值104857600 - 默认值104857600

View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisBaseSerializer {
}

View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisCsvSerializer {
}

View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisDelimiterParser {
}

View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisJsonSerializer {
}

View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisSerializer {
}

View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisSerializerFactory {
}

View File

@ -0,0 +1,2 @@
package com.alibaba.datax.plugin.writer.doriswriter;public class DorisStreamLoadExcetion {
}

View File

@ -7,6 +7,7 @@
"preSql": [], "preSql": [],
"postSql": [], "postSql": [],
"beLoadUrl": [], "beLoadUrl": [],
"loadUrl": [],
"loadProps": {}, "loadProps": {},
"connection": [ "connection": [
{ {