From 65a880b852cec83a6313e4cbb09a1cfa70a80ed6 Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Tue, 9 May 2023 12:10:55 +0800 Subject: [PATCH 01/16] [feature][plugin][sybase] datax support sybase plugins. #1780 add doc --- sybasereader/doc/sybasereader.md | 327 +++++++++++++++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 sybasereader/doc/sybasereader.md diff --git a/sybasereader/doc/sybasereader.md b/sybasereader/doc/sybasereader.md new file mode 100644 index 00000000..32851e52 --- /dev/null +++ b/sybasereader/doc/sybasereader.md @@ -0,0 +1,327 @@ + +# SybaseReader 插件文档 + + +___ + + +## 1 快速介绍 + +SybaseReader插件实现了从Sybase读取数据。在底层实现上，SybaseReader通过JDBC连接远程Sybase数据库，并执行相应的sql语句将数据从Sybase库中SELECT出来。 + +## 2 实现原理 + +简而言之，SybaseReader通过JDBC连接器连接到远程的Sybase数据库，并根据用户配置的信息生成查询SELECT SQL语句并发送到远程Sybase数据库，并将该SQL执行返回结果使用DataX自定义的数据类型拼装为抽象的数据集，并传递给下游Writer处理。 + +对于用户配置Table、Column、Where的信息，SybaseReader将其拼接为SQL语句发送到Sybase数据库；对于用户配置querySql信息，Sybase直接将其发送到Sybase数据库。 + + +## 3 功能说明 + +### 3.1 配置样例 + +* 配置一个从Sybase数据库同步抽取数据到本地的作业: + +``` +{ + "job": { + "setting": { + "speed": { + //设置传输速度 byte/s 尽量逼近这个速度但是不高于它. + // channel 表示通道数量，byte表示通道速度，如果单通道速度1MB，配置byte为1048576表示一个channel + "byte": 1048576 + }, + //出错限制 + "errorLimit": { + //先选择record + "record": 0, + //百分比 1表示100% + "percentage": 0.02 + } + }, + "content": [ + { + "reader": { + "name": "SybaseReader", + "parameter": { + // 数据库连接用户名 + "username": "root", + // 数据库连接密码 + "password": "root", + "column": [ + "id","name" + ], + //切分主键 + "splitPk": "db_id", + "connection": [ + { + "table": [ + "table" + ], + "jdbcUrl": [ + "jdbc:Sybase:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]" + ] + } + ] + } + }, + "writer": { + //writer类型 + "name": "streamwriter", + // 是否打印内容 + "parameter": { + "print": true + } + } + } + ] + } +} + +``` + +* 配置一个自定义SQL的数据库同步任务到本地内容的作业： + +``` +{ + "job": { + "setting": { + "speed": { + "channel": 5 + } + }, + "content": [ + { + "reader": { + "name": "SybaseReader", + "parameter": { + "username": "root", + "password": "root", + "where": "", + "connection": [ + { + "querySql": [ + "select db_id,on_line_flag from db_info where db_id < 10" + ], + "jdbcUrl": [ + "jdbc:Sybase:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]" + ] + } + ] + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "visible": false, + "encoding": "UTF-8" + } + } + } + ] + } +} +``` + + +### 3.2 参数说明 + +* **jdbcUrl** + + * 描述：描述的是到对端数据库的JDBC连接信息，使用JSON的数组描述，并支持一个库填写多个连接地址。之所以使用JSON数组描述连接信息，是因为阿里集团内部支持多个IP探测，如果配置了多个，SybaseReader可以依次探测ip的可连接性，直到选择一个合法的IP。如果全部连接失败，SybaseReader报错。 注意，jdbcUrl必须包含在connection配置单元中。对于阿里集团外部使用情况，JSON数组填写一个JDBC连接即可。 + + jdbcUrl按照Sybase官方规范，并可以填写连接附件控制信息。具体请参看[Sybase官方文档](http://www.Sybase.com/technetwork/database/enterprise-edition/documentation/index.html)。 + + * 必选：是
+ + * 默认值：无
+ +* **username** + + * 描述：数据源的用户名
+ + * 必选：是
+ + * 默认值：无
+ +* **password** + + * 描述：数据源指定用户名的密码
+ + * 必选：是
+ + * 默认值：无
+ +* **table** + + * 描述：所选取的需要同步的表。使用JSON的数组描述，因此支持多张表同时抽取。当配置为多张表时，用户自己需保证多张表是同一schema结构，SybaseReader不予检查表是否同一逻辑表。注意，table必须包含在connection配置单元中。
+ + * 必选：是
+ + * 默认值：无
+ +* **column** + + * 描述：所配置的表中需要同步的列名集合，使用JSON的数组描述字段信息。用户使用\*代表默认使用所有列配置，例如['\*']。 + + 支持列裁剪，即列可以挑选部分列进行导出。 + + 支持列换序，即列可以不按照表schema信息进行导出。 + + 支持常量配置，用户需要按照JSON格式: + ["id", "`table`", "1", "'bazhen.csy'", "null", "to_char(a + 1)", "2.3" , "true"] + id为普通列名，\`table\`为包含保留在的列名，1为整形数字常量，'bazhen.csy'为字符串常量，null为空指针，to_char(a + 1)为表达式，2.3为浮点数，true为布尔值。 + + Column必须显示填写，不允许为空！ + + * 必选：是
+ + * 默认值：无
+ +* **splitPk** + + * 描述：SybaseReader进行数据抽取时，如果指定splitPk，表示用户希望使用splitPk代表的字段进行数据分片，DataX因此会启动并发任务进行数据同步，这样可以大大提供数据同步的效能。 + + 推荐splitPk用户使用表主键，因为表主键通常情况下比较均匀，因此切分出来的分片也不容易出现数据热点。 + + 目前splitPk仅支持整形、字符串型数据切分，`不支持浮点、日期等其他类型`。如果用户指定其他非支持类型，SybaseReader将报错！ + + splitPk如果不填写，将视作用户不对单表进行切分，SybaseReader使用单通道同步全量数据。 + + * 必选：否
+ + * 默认值：无
+ +* **where** + + * 描述：筛选条件，MysqlReader根据指定的column、table、where条件拼接SQL，并根据这个SQL进行数据抽取。在实际业务场景中，往往会选择当天的数据进行同步，可以将where条件指定为gmt_create > $bizdate 。注意：不可以将where条件指定为limit 10，limit不是SQL的合法where子句。
+ + where条件可以有效地进行业务增量同步。 + + * 必选：否
+ + * 默认值：无
+ +* **querySql** + + * 描述：在有些业务场景下，where这一配置项不足以描述所筛选的条件，用户可以通过该配置型来自定义筛选SQL。当用户配置了这一项之后，DataX系统就会忽略table，column这些配置型，直接使用这个配置项的内容对数据进行筛选，例如需要进行多表join后同步数据，使用select a,b from table_a join table_b on table_a.id = table_b.id
+ + `当用户配置querySql时，SybaseReader直接忽略table、column、where条件的配置`。 + + * 必选：否
+ + * 默认值：无
+ +* **fetchSize** + + * 描述：该配置项定义了插件和数据库服务器端每次批量数据获取条数，该值决定了DataX和服务器端的网络交互次数，能够较大的提升数据抽取性能。
+ + `注意，该值过大(>2048)可能造成DataX进程OOM。`。 + + * 必选：否
+ + * 默认值：1024
+ + + +### 3.3 类型转换 + +目前SybaseReader支持大部分Sybase类型，但也存在部分个别类型没有支持的情况，请注意检查你的类型。 + +下面列出SybaseReader针对Sybase类型转换列表: + + +| DataX 内部类型| Sybase 数据类型 | +| -------- | ----- | +| Long |NUMBER,INTEGER,INT,SMALLINT| +| Double |NUMERIC,DECIMAL,FLOAT,DOUBLE PRECISION,REAL| +| String |LONG,CHAR,NCHAR,VARCHAR,VARCHAR2,NVARCHAR2,CLOB,NCLOB,CHARACTER,CHARACTER VARYING,CHAR VARYING,NATIONAL CHARACTER,NATIONAL CHAR,NATIONAL CHARACTER VARYING,NATIONAL CHAR VARYING,NCHAR VARYING | +| Date |TIMESTAMP,DATE | +| Boolean |bit, bool | +| Bytes |BLOB,BFILE,RAW,LONG RAW | + + + +请注意: + +* `除上述罗列字段类型外，其他类型均不支持`。 + + +## 4 性能报告 + +### 4.1 环境准备 + +#### 4.1.1 数据特征 + +为了模拟线上真实数据，我们设计两个Sybase数据表，分别为: + +#### 4.1.2 机器参数 + +* 执行DataX的机器参数为: + +* Sybase数据库机器参数为: + +### 4.2 测试报告 + +#### 4.2.1 表1测试报告 + + +| 并发任务数| DataX速度(Rec/s)|DataX流量|网卡流量|DataX运行负载|DB运行负载| +|--------| --------|--------|--------|--------|--------| +|1| DataX 统计速度(Rec/s)|DataX统计流量|网卡流量|DataX运行负载|DB运行负载| + +## 5 约束限制 + + +### 5.1 一致性约束 + +Sybase在数据存储划分中属于RDBMS系统，对外可以提供强一致性数据查询接口。例如当一次同步任务启动运行过程中，当该库存在其他数据写入方写入数据时，SybaseReader完全不会获取到写入更新数据，这是由于数据库本身的快照特性决定的。关于数据库快照特性，请参看[MVCC Wikipedia](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) + +上述是在SybaseReader单线程模型下数据同步一致性的特性，由于SybaseReader可以根据用户配置信息使用了并发数据抽取，因此不能严格保证数据一致性：当SybaseReader根据splitPk进行数据切分后，会先后启动多个并发任务完成数据同步。由于多个并发任务相互之间不属于同一个读事务，同时多个并发任务存在时间间隔。因此这份数据并不是`完整的`、`一致的`数据快照信息。 + +针对多线程的一致性快照需求，在技术上目前无法实现，只能从工程角度解决，工程化的方式存在取舍，我们提供几个解决思路给用户，用户可以自行选择： + +1. 使用单线程同步，即不再进行数据切片。缺点是速度比较慢，但是能够很好保证一致性。 + +2. 关闭其他数据写入方，保证当前数据为静态数据，例如，锁表、关闭备库同步等等。缺点是可能影响在线业务。 + +### 5.2 数据库编码问题 + + +SybaseReader底层使用JDBC进行数据抽取，JDBC天然适配各类编码，并在底层进行了编码转换。因此SybaseReader不需用户指定编码，可以自动获取编码并转码。 + +对于Sybase底层写入编码和其设定的编码不一致的混乱情况，SybaseReader对此无法识别，对此也无法提供解决方案，对于这类情况，`导出有可能为乱码`。 + +### 5.3 增量数据同步 + +SybaseReader使用JDBC SELECT语句完成数据抽取工作，因此可以使用SELECT...WHERE...进行增量数据抽取，方式有多种： + +* 数据库在线应用写入数据库时，填充modify字段为更改时间戳，包括新增、更新、删除(逻辑删)。对于这类应用，SybaseReader只需要WHERE条件跟上一同步阶段时间戳即可。 +* 对于新增流水型数据，SybaseReader可以WHERE条件后跟上一阶段最大自增ID即可。 + +对于业务上无字段区分新增、修改数据情况，SybaseReader也无法进行增量数据同步，只能同步全量数据。 + +### 5.4 Sql安全性 + +SybaseReader提供querySql语句交给用户自己实现SELECT抽取语句，SybaseReader本身对querySql不做任何安全性校验。这块交由DataX用户方自己保证。 + +## 6 FAQ + +*** + +**Q: 目前已验证支持sybase的版本？** + + A: Sybase ASE 16/15.7 + +**Q: SybaseReader同步报错，报错信息为XXX** + + A: 网络或者权限问题，请使用Sybase命令行或者可视化工具进行测试： + 如果上述命令也报错，那可以证实是环境问题，请联系你的DBA。 + + +**Q: SybaseReader抽取速度很慢怎么办？** + + A: 影响抽取时间的原因大概有如下几个： + 1. 由于SQL的plan异常，导致的抽取时间长； 在抽取时，尽可能使用全表扫描代替索引扫描; + 2. 合理sql的并发度，减少抽取时间；根据表的大小， + 3. 设置合理fetchsize，减少网络IO; From e158e5845bf324849dc08e1e3c2c648d06fbf0bd Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Tue, 9 May 2023 17:09:23 +0800 Subject: [PATCH 02/16] [feature][plugin][sybase] datax support sybase plugins. #1780 add reader plugin --- .../datax/plugin/rdbms/util/DataBaseType.java | 4 +- sybasereader/doc/sybasereader.md | 12 +- sybasereader/pom.xml | 105 +++++++++++++++++ sybasereader/src/main/assembly/package.xml | 35 ++++++ .../plugin/reader/sybasereader/Constants.java | 7 ++ .../reader/sybasereader/SybaseReader.java | 107 ++++++++++++++++++ sybasereader/src/main/resources/plugin.json | 6 + .../main/resources/plugin_job_template.json | 14 +++ sybasewriter/pom.xml | 19 ++++ 9 files changed, 302 insertions(+), 7 deletions(-) create mode 100644 sybasereader/pom.xml create mode 100755 sybasereader/src/main/assembly/package.xml create mode 100755 sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/Constants.java create mode 100755 sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseReader.java create mode 100755 sybasereader/src/main/resources/plugin.json create mode 100644 sybasereader/src/main/resources/plugin_job_template.json create mode 100644 sybasewriter/pom.xml diff --git a/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/DataBaseType.java b/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/DataBaseType.java index 1b46a8bc..e33ea1a6 100755 --- a/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/DataBaseType.java +++ b/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/DataBaseType.java @@ -25,7 +25,9 @@ public enum DataBaseType { Oscar("oscar", "com.oscar.Driver"), OceanBase("oceanbase", "com.alipay.oceanbase.jdbc.Driver"), StarRocks("starrocks", "com.mysql.jdbc.Driver"), - Databend("databend", "com.databend.jdbc.DatabendDriver"); + Databend("databend", "com.databend.jdbc.DatabendDriver"), + Sybase("sybase", "com.sybase.jdbc4.jdbc.SybDriver"); + private String typeName; private String driverClassName; diff --git a/sybasereader/doc/sybasereader.md b/sybasereader/doc/sybasereader.md index 32851e52..54156706 100644 --- a/sybasereader/doc/sybasereader.md +++ b/sybasereader/doc/sybasereader.md @@ -233,12 +233,12 @@ SybaseReader插件实现了从Sybase读取数据。在底层实现上，SybaseRe | DataX 内部类型| Sybase 数据类型 | | -------- | ----- | -| Long |NUMBER,INTEGER,INT,SMALLINT| -| Double |NUMERIC,DECIMAL,FLOAT,DOUBLE PRECISION,REAL| -| String |LONG,CHAR,NCHAR,VARCHAR,VARCHAR2,NVARCHAR2,CLOB,NCLOB,CHARACTER,CHARACTER VARYING,CHAR VARYING,NATIONAL CHARACTER,NATIONAL CHAR,NATIONAL CHARACTER VARYING,NATIONAL CHAR VARYING,NCHAR VARYING | -| Date |TIMESTAMP,DATE | -| Boolean |bit, bool | -| Bytes |BLOB,BFILE,RAW,LONG RAW | +| Long |Tinyint,Smallint,Int,Money,Smallmoney| +| Double |Float,Real,Numeric,Decimal| +| String |Char,Varchar,Nchar,Nvarchar,Text| +| Date |Timestamp,Datetime,Smalldatetime| +| Boolean |bit, bool| +| Bytes |Binary,Varbinary,Image| diff --git a/sybasereader/pom.xml b/sybasereader/pom.xml new file mode 100644 index 00000000..1c826fb4 --- /dev/null +++ b/sybasereader/pom.xml @@ -0,0 +1,105 @@ + + + + datax-all + com.alibaba.datax + 0.0.1-SNAPSHOT + + 4.0.0 + + sybasereader + sybasereader + jar + + + 8 + 8 + + + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + + com.alibaba.datax + plugin-rdbms-util + ${datax-project-version} + + + + com.oracle + ojdbc6 + 11.2.0.3 + + + com.alibaba.datax + datax-common + 0.0.1-SNAPSHOT + compile + + + + com.sybase.jconnect + jconn4 + 16.0 + system + ${project.basedir}/libs/jconn4-16.0.jar + + + + + + + + + + maven-compiler-plugin + + ${jdk-version} + ${jdk-version} + ${project-sourceEncoding} + + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + + + \ No newline at end of file diff --git a/sybasereader/src/main/assembly/package.xml b/sybasereader/src/main/assembly/package.xml new file mode 100755 index 00000000..a954a30d --- /dev/null +++ b/sybasereader/src/main/assembly/package.xml @@ -0,0 +1,35 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/reader/oraclereader + + + target/ + + oraclereader-0.0.1-SNAPSHOT.jar + + plugin/reader/oraclereader + + + + + + false + plugin/reader/oraclereader/libs + runtime + + + diff --git a/sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/Constants.java b/sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/Constants.java new file mode 100755 index 00000000..2de97644 --- /dev/null +++ b/sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/Constants.java @@ -0,0 +1,7 @@ +package com.alibaba.datax.plugin.reader.sybasereader; + +public class Constants { + + public static final int DEFAULT_FETCH_SIZE = 1024; + +} diff --git a/sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseReader.java b/sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseReader.java new file mode 100755 index 00000000..d0bbefac --- /dev/null +++ b/sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseReader.java @@ -0,0 +1,107 @@ +package com.alibaba.datax.plugin.reader.sybasereader; + +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.common.spi.Reader; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.plugin.rdbms.util.DataBaseType; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import com.alibaba.datax.plugin.rdbms.reader.CommonRdbmsReader; +import com.alibaba.datax.plugin.rdbms.reader.Constant; + + +public class SybaseReader extends Reader { + + private static final DataBaseType DATABASE_TYPE = DataBaseType.Oracle; + + public static class Job extends Reader.Job { + private static final Logger LOG = LoggerFactory + .getLogger(SybaseReader.Job.class); + + private Configuration originalConfig = null; + private CommonRdbmsReader.Job commonRdbmsReaderJob; + + @Override + public void init() { + this.originalConfig = super.getPluginJobConf(); + + dealFetchSize(this.originalConfig); + + this.commonRdbmsReaderJob = new CommonRdbmsReader.Job( + DATABASE_TYPE); + this.commonRdbmsReaderJob.init(this.originalConfig); + + } + + @Override + public void preCheck(){ + init(); + this.commonRdbmsReaderJob.preCheck(this.originalConfig,DATABASE_TYPE); + } + + @Override + public List split(int adviceNumber) { + return this.commonRdbmsReaderJob.split(this.originalConfig, + adviceNumber); + } + + @Override + public void post() { + this.commonRdbmsReaderJob.post(this.originalConfig); + } + + @Override + public void destroy() { + this.commonRdbmsReaderJob.destroy(this.originalConfig); + } + + private void dealFetchSize(Configuration originalConfig) { + int fetchSize = originalConfig.getInt( + com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE, + Constants.DEFAULT_FETCH_SIZE); + if (fetchSize < 1) { + } + originalConfig.set( + com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE, + fetchSize); + } + } + + public static class Task extends Reader.Task { + + private Configuration readerSliceConfig; + private CommonRdbmsReader.Task commonRdbmsReaderTask; + + @Override + public void init() { + this.readerSliceConfig = super.getPluginJobConf(); + this.commonRdbmsReaderTask = new CommonRdbmsReader.Task( + DATABASE_TYPE ,super.getTaskGroupId(), super.getTaskId()); + this.commonRdbmsReaderTask.init(this.readerSliceConfig); + } + + @Override + public void startRead(RecordSender recordSender) { + int fetchSize = this.readerSliceConfig + .getInt(com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE); + + this.commonRdbmsReaderTask.startRead(this.readerSliceConfig, + recordSender, super.getTaskPluginCollector(), fetchSize); + } + + @Override + public void post() { + this.commonRdbmsReaderTask.post(this.readerSliceConfig); + } + + @Override + public void destroy() { + this.commonRdbmsReaderTask.destroy(this.readerSliceConfig); + } + + } + +} diff --git a/sybasereader/src/main/resources/plugin.json b/sybasereader/src/main/resources/plugin.json new file mode 100755 index 00000000..39dd61d7 --- /dev/null +++ b/sybasereader/src/main/resources/plugin.json @@ -0,0 +1,6 @@ +{ + "name": "sybasereader", + "class": "com.alibaba.datax.plugin.reader.sybasereader.SybaseReader", + "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", + "developer": "alibaba" +} \ No newline at end of file diff --git a/sybasereader/src/main/resources/plugin_job_template.json b/sybasereader/src/main/resources/plugin_job_template.json new file mode 100644 index 00000000..5d5a1f45 --- /dev/null +++ b/sybasereader/src/main/resources/plugin_job_template.json @@ -0,0 +1,14 @@ +{ + "name": "sybasereader", + "parameter": { + "username": "", + "password": "", + "column": [], + "connection": [ + { + "table": [], + "jdbcUrl": [] + } + ] + } +} \ No newline at end of file diff --git a/sybasewriter/pom.xml b/sybasewriter/pom.xml new file mode 100644 index 00000000..f077f52d --- /dev/null +++ b/sybasewriter/pom.xml @@ -0,0 +1,19 @@ + + + + datax-all + com.alibaba.datax + 0.0.1-SNAPSHOT + + 4.0.0 + + sybasewriter + + + 8 + 8 + + + \ No newline at end of file From 543a67273f70bd434ef6352cb78c702d63d2b664 Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Tue, 9 May 2023 17:20:23 +0800 Subject: [PATCH 03/16] [feature][plugin][sybase] datax support sybase plugins. #1780 update package --- sybasereader/src/main/assembly/package.xml | 8 ++++---- .../datax/plugin/reader/sybasereader/SybaseReader.java | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sybasereader/src/main/assembly/package.xml b/sybasereader/src/main/assembly/package.xml index a954a30d..40060050 100755 --- a/sybasereader/src/main/assembly/package.xml +++ b/sybasereader/src/main/assembly/package.xml @@ -14,21 +14,21 @@ plugin.json plugin_job_template.json - plugin/reader/oraclereader + plugin/reader/sybasereader target/ - oraclereader-0.0.1-SNAPSHOT.jar + sybasereader-0.0.1-SNAPSHOT.jar - plugin/reader/oraclereader + plugin/reader/sybasereader false - plugin/reader/oraclereader/libs + plugin/reader/sybasereader/libs runtime diff --git a/sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseReader.java b/sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseReader.java index d0bbefac..f0a0ac1a 100755 --- a/sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseReader.java +++ b/sybasereader/src/main/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseReader.java @@ -15,7 +15,7 @@ import com.alibaba.datax.plugin.rdbms.reader.Constant; public class SybaseReader extends Reader { - private static final DataBaseType DATABASE_TYPE = DataBaseType.Oracle; + private static final DataBaseType DATABASE_TYPE = DataBaseType.Sybase; public static class Job extends Reader.Job { private static final Logger LOG = LoggerFactory @@ -63,6 +63,7 @@ public class SybaseReader extends Reader { com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE, Constants.DEFAULT_FETCH_SIZE); if (fetchSize < 1) { + LOG.warn("对 sybasereader 需要配置 fetchSize, 对性能提升有较大影响 请配置fetchSize."); } originalConfig.set( com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE, From 5b9c58c4f8c7d751d327d015fad61da19095f806 Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Wed, 10 May 2023 10:00:03 +0800 Subject: [PATCH 04/16] [feature][plugin][sybase] datax support sybase plugins. #1780 update reader markdown --- sybasereader/doc/sybasereader.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sybasereader/doc/sybasereader.md b/sybasereader/doc/sybasereader.md index 54156706..abde7cb1 100644 --- a/sybasereader/doc/sybasereader.md +++ b/sybasereader/doc/sybasereader.md @@ -59,7 +59,7 @@ SybaseReader插件实现了从Sybase读取数据。在底层实现上，SybaseRe "table" ], "jdbcUrl": [ - "jdbc:Sybase:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]" + "jdbc:sybase:Tds:192.168.1.92:5000/tempdb?charset=cp936" ] } ] @@ -104,7 +104,7 @@ SybaseReader插件实现了从Sybase读取数据。在底层实现上，SybaseRe "select db_id,on_line_flag from db_info where db_id < 10" ], "jdbcUrl": [ - "jdbc:Sybase:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]" + "jdbc:sybase:Tds:192.168.1.92:5000/tempdb?charset=cp936" ] } ] From 49c7d5618bf0f16d4f0f872429a626f7f6878af6 Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Wed, 10 May 2023 10:09:03 +0800 Subject: [PATCH 05/16] [feature][plugin][sybase] datax support sybase plugins. #1780 add sybasewriter markdown --- sybasewriter/doc/sybasewriter.md | 228 +++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 sybasewriter/doc/sybasewriter.md diff --git a/sybasewriter/doc/sybasewriter.md b/sybasewriter/doc/sybasewriter.md new file mode 100644 index 00000000..cccc62d6 --- /dev/null +++ b/sybasewriter/doc/sybasewriter.md @@ -0,0 +1,228 @@ +# DataX SybaseWriter + + +--- + + +## 1 快速介绍 + +SybaseWriter 插件实现了写入数据到 Sybase 主库的目的表的功能。在底层实现上， SybaseWriter 通过 JDBC 连接远程 Sybase 数据库，并执行相应的 insert into ... 或者 ( replace into ...) 的 sql 语句将数据写入 Sybase，内部会分批次提交入库，需要数据库本身采用 innodb 引擎。 + +SybaseWriter 面向ETL开发工程师，他们使用 SybaseWriter 从数仓导入数据到 Sybase。同时 SybaseWriter 亦可以作为数据迁移工具为DBA等用户提供服务。 + + +## 2 实现原理 + +SybaseWriter 通过 DataX 框架获取 Reader 生成的协议数据，根据你配置的 `writeMode` 生成 + + +* `insert into...`(当主键/唯一性索引冲突时会写不进去冲突的行) + +##### 或者 + +* `replace into...`(没有遇到主键/唯一性索引冲突时，与 insert into 行为一致，冲突时会用新行替换原有行所有字段) 的语句写入数据到 Sybase。出于性能考虑，采用了 `PreparedStatement + Batch`，并且设置了：`rewriteBatchedStatements=true`，将数据缓冲到线程上下文 Buffer 中，当 Buffer 累计到预定阈值时，才发起写入请求。 + +
+ + 注意：目的表所在数据库必须是主库才能写入数据；整个任务至少需要具备 insert/replace into...的权限，是否需要其他权限，取决于你任务配置中在 preSql 和 postSql 中指定的语句。 + + +## 3 功能说明 + +### 3.1 配置样例 + +* 这里使用一份从内存产生到 Sybase 导入的数据。 + +```json +{ + "job": { + "setting": { + "speed": { + "channel": 1 + } + }, + "content": [ + { + "reader": { + "name": "streamreader", + "parameter": { + "column" : [ + { + "value": "DataX", + "type": "string" + }, + { + "value": 19880808, + "type": "long" + }, + { + "value": "1988-08-08 08:08:08", + "type": "date" + }, + { + "value": true, + "type": "bool" + }, + { + "value": "test", + "type": "bytes" + } + ], + "sliceRecordCount": 1000 + } + }, + "writer": { + "name": "Sybasewriter", + "parameter": { + "writeMode": "insert", + "username": "root", + "password": "root", + "column": [ + "id", + "name" + ], + "preSql": [ + "delete from test" + ], + "connection": [ + { + "jdbcUrl":"jdbc:sybase:Tds:192.168.1.92:5000/tempdb?charset=cp936", + "table": [ + "test" + ] + } + ] + } + } + } + ] + } +} + +``` + + +### 3.2 参数说明 + +* **jdbcUrl** + + * 描述：目的数据库的 JDBC 连接信息。作业运行时，DataX 会在你提供的 jdbcUrl 后面追加如下属性：yearIsDateType=false&zeroDateTimeBehavior=convertToNull&rewriteBatchedStatements=true + + 注意：1、在一个数据库上只能配置一个 jdbcUrl 值。这与 SybaseReader 支持多个备库探测不同，因为此处不支持同一个数据库存在多个主库的情况(双主导入数据情况) + 2、jdbcUrl按照Sybase官方规范，并可以填写连接附加控制信息，比如想指定连接编码为 gbk ，则在 jdbcUrl 后面追加属性 useUnicode=true&characterEncoding=gbk。具体请参看 Sybase官方文档或者咨询对应 DBA。 + + + * 必选：是
+ + * 默认值：无
+ +* **username** + + * 描述：目的数据库的用户名
+ + * 必选：是
+ + * 默认值：无
+ +* **password** + + * 描述：目的数据库的密码
+ + * 必选：是
+ + * 默认值：无
+ +* **table** + + * 描述：目的表的表名称。支持写入一个或者多个表。当配置为多张表时，必须确保所有表结构保持一致。 + + 注意：table 和 jdbcUrl 必须包含在 connection 配置单元中 + + * 必选：是
+ + * 默认值：无
+ +* **column** + + * 描述：目的表需要写入数据的字段,字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。如果要依次写入全部列，使用`*`表示, 例如: `"column": ["*"]`。 + + **column配置项必须指定，不能留空！** + + 注意：1、我们强烈不推荐你这样配置，因为当你目的表字段个数、类型等有改动时，你的任务可能运行不正确或者失败 + 2、 column 不能配置任何常量值 + + * 必选：是
+ + * 默认值：否
+ +* **preSql** + + * 描述：写入数据到目的表前，会先执行这里的标准语句。如果 Sql 中有你需要操作到的表名称，请使用 `@table` 表示，这样在实际执行 Sql 语句时，会对变量按照实际表名称进行替换。比如你的任务是要写入到目的端的100个同构分表(表名称为:datax_00,datax01, ... datax_98,datax_99)，并且你希望导入数据前，先对表中数据进行删除操作，那么你可以这样配置：`"preSql":["delete from 表名"]`，效果是：在执行到每个表写入数据前，会先执行对应的 delete from 对应表名称
+ + * 必选：否
+ + * 默认值：无
+ +* **postSql** + + * 描述：写入数据到目的表后，会执行这里的标准语句。（原理同 preSql ）
+ + * 必选：否
+ + * 默认值：无
+ +* **writeMode** + + * 描述：控制写入数据到目标表采用 `insert into` 或者 `replace into` 或者 `ON DUPLICATE KEY UPDATE` 语句
+ + * 必选：是
+ + * 所有选项：insert/replace/update
+ + * 默认值：insert
+ +* **batchSize** + + * 描述：一次性批量提交的记录数大小，该值可以极大减少DataX与Sybase的网络交互次数，并提升整体吞吐量。但是该值设置过大可能会造成DataX运行进程OOM情况。
+ + * 必选：否
+ + * 默认值：1024
+ + +### 3.3 类型转换 + +类似 SybaseReader ，目前 SybaseWriter 支持大部分 Sybase 类型，但也存在部分个别类型没有支持的情况，请注意检查你的类型。 + +下面列出 SybaseWriter 针对 Sybase 类型转换列表: + + +| DataX 内部类型| Sybase 数据类型 | +| -------- | ----- | +| Long |Tinyint,Smallint,Int,Money,Smallmoney| +| Double |Float,Real,Numeric,Decimal| +| String |Char,Varchar,Nchar,Nvarchar,Text| +| Date |Timestamp,Datetime,Smalldatetime| +| Boolean |bit, bool| +| Bytes |Binary,Varbinary,Image| + +## 4 性能报告 + + +## 5 约束限制 + + + + +## FAQ + +*** + +**Q: 目前已验证支持sybase的版本？** + +A: Sybase ASE 16/15.7 + +**Q: SybaseReader同步报错，报错信息为XXX** + +A: 网络或者权限问题，请使用Sybase命令行或者可视化工具进行测试： +如果上述命令也报错，那可以证实是环境问题，请联系你的DBA。 From a1e62f02461b58c5eb9d09546bbfb32bf3bc8e1d Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Wed, 10 May 2023 10:09:30 +0800 Subject: [PATCH 06/16] [feature][plugin][sybase] datax support sybase plugins. #1780 add sybase pom.xml --- sybasewriter/pom.xml | 81 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/sybasewriter/pom.xml b/sybasewriter/pom.xml index f077f52d..c5b57549 100644 --- a/sybasewriter/pom.xml +++ b/sybasewriter/pom.xml @@ -16,4 +16,85 @@ 8 + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + + com.alibaba.datax + plugin-rdbms-util + ${datax-project-version} + + + + com.oracle + ojdbc6 + 11.2.0.3 + + + com.alibaba.datax + datax-common + 0.0.1-SNAPSHOT + compile + + + + com.sybase.jconnect + jconn4 + 16.0 + system + ${project.basedir}/libs/jconn4-16.0.jar + + + + + + + + maven-compiler-plugin + + ${jdk-version} + ${jdk-version} + ${project-sourceEncoding} + + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + \ No newline at end of file From db7988b1f674394a2694b243f96ca09ee06e3f7c Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Wed, 10 May 2023 10:13:28 +0800 Subject: [PATCH 07/16] [feature][plugin][sybase] datax support sybase plugins. #1780 add writer package.xml --- .../src/main/java/assembly/package.xml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 sybasewriter/src/main/java/assembly/package.xml diff --git a/sybasewriter/src/main/java/assembly/package.xml b/sybasewriter/src/main/java/assembly/package.xml new file mode 100755 index 00000000..511c5627 --- /dev/null +++ b/sybasewriter/src/main/java/assembly/package.xml @@ -0,0 +1,35 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/reader/sybasewriter + + + target/ + + sybasewriter-0.0.1-SNAPSHOT.jar + + plugin/reader/sybasewriter + + + + + + false + plugin/reader/sybasewriter/libs + runtime + + + From d831c1665154e745805f6156214771fefc73f8e4 Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Wed, 10 May 2023 10:15:20 +0800 Subject: [PATCH 08/16] [feature][plugin][sybase] datax support sybase plugins. #1780 add resources --- sybasewriter/src/main/java/resources/plugin.json | 6 ++++++ .../main/java/resources/plugin_job_template.json | 14 ++++++++++++++ 2 files changed, 20 insertions(+) create mode 100755 sybasewriter/src/main/java/resources/plugin.json create mode 100644 sybasewriter/src/main/java/resources/plugin_job_template.json diff --git a/sybasewriter/src/main/java/resources/plugin.json b/sybasewriter/src/main/java/resources/plugin.json new file mode 100755 index 00000000..6bfa66f3 --- /dev/null +++ b/sybasewriter/src/main/java/resources/plugin.json @@ -0,0 +1,6 @@ +{ + "name": "sybasewriter", + "class": "com.alibaba.datax.plugin.reader.sybasewriter.SybaseWriter", + "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", + "developer": "alibaba" +} \ No newline at end of file diff --git a/sybasewriter/src/main/java/resources/plugin_job_template.json b/sybasewriter/src/main/java/resources/plugin_job_template.json new file mode 100644 index 00000000..212f76b9 --- /dev/null +++ b/sybasewriter/src/main/java/resources/plugin_job_template.json @@ -0,0 +1,14 @@ +{ + "name": "sybasewriter", + "parameter": { + "username": "", + "password": "", + "column": [], + "connection": [ + { + "table": [], + "jdbcUrl": [] + } + ] + } +} \ No newline at end of file From e1262dc028fa83eb0820a63fd85408a20476f5a6 Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Wed, 10 May 2023 10:23:31 +0800 Subject: [PATCH 09/16] [feature][plugin][sybase] datax support sybase plugins. #1780 add sybasewriter --- .../reader/sybasewriter/SybaseWriter.java | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100755 sybasewriter/src/main/java/com/alibaba/datax/plugin/reader/sybasewriter/SybaseWriter.java diff --git a/sybasewriter/src/main/java/com/alibaba/datax/plugin/reader/sybasewriter/SybaseWriter.java b/sybasewriter/src/main/java/com/alibaba/datax/plugin/reader/sybasewriter/SybaseWriter.java new file mode 100755 index 00000000..220e6233 --- /dev/null +++ b/sybasewriter/src/main/java/com/alibaba/datax/plugin/reader/sybasewriter/SybaseWriter.java @@ -0,0 +1,100 @@ +package com.alibaba.datax.plugin.reader.sybasewriter; + +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.spi.Writer; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.plugin.rdbms.util.DataBaseType; +import com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter; +import com.alibaba.datax.plugin.rdbms.writer.Key; + +import java.util.List; + + +public class SybaseWriter extends Writer { + private static final DataBaseType DATABASE_TYPE = DataBaseType.SYBASE; + + public static class Job extends Writer.Job { + private Configuration originalConfig = null; + private CommonRdbmsWriter.Job commonRdbmsWriterJob; + + @Override + public void preCheck(){ + this.init(); + this.commonRdbmsWriterJob.writerPreCheck(this.originalConfig, DATABASE_TYPE); + } + + @Override + public void init() { + this.originalConfig = super.getPluginJobConf(); + this.commonRdbmsWriterJob = new CommonRdbmsWriter.Job(DATABASE_TYPE); + this.commonRdbmsWriterJob.init(this.originalConfig); + } + + // 一般来说，是需要推迟到 task 中进行pre 的执行（单表情况例外） + @Override + public void prepare() { + //实跑先不支持 权限 检验 + //this.commonRdbmsWriterJob.privilegeValid(this.originalConfig, DATABASE_TYPE); + this.commonRdbmsWriterJob.prepare(this.originalConfig); + } + + @Override + public List split(int mandatoryNumber) { + return this.commonRdbmsWriterJob.split(this.originalConfig, mandatoryNumber); + } + + // 一般来说，是需要推迟到 task 中进行post 的执行（单表情况例外） + @Override + public void post() { + this.commonRdbmsWriterJob.post(this.originalConfig); + } + + @Override + public void destroy() { + this.commonRdbmsWriterJob.destroy(this.originalConfig); + } + + } + + public static class Task extends Writer.Task { + private Configuration writerSliceConfig; + private CommonRdbmsWriter.Task commonRdbmsWriterTask; + + @Override + public void init() { + this.writerSliceConfig = super.getPluginJobConf(); + this.commonRdbmsWriterTask = new CommonRdbmsWriter.Task(DATABASE_TYPE); + this.commonRdbmsWriterTask.init(this.writerSliceConfig); + } + + @Override + public void prepare() { + this.commonRdbmsWriterTask.prepare(this.writerSliceConfig); + } + + //TODO 改用连接池，确保每次获取的连接都是可用的（注意：连接可能需要每次都初始化其 session） + public void startWrite(RecordReceiver recordReceiver) { + this.commonRdbmsWriterTask.startWrite(recordReceiver, this.writerSliceConfig, + super.getTaskPluginCollector()); + } + + @Override + public void post() { + this.commonRdbmsWriterTask.post(this.writerSliceConfig); + } + + @Override + public void destroy() { + this.commonRdbmsWriterTask.destroy(this.writerSliceConfig); + } + + @Override + public boolean supportFailOver(){ + String writeMode = writerSliceConfig.getString(Key.WRITE_MODE); + return "replace".equalsIgnoreCase(writeMode); + } + + } + + +} From e03f869731da3cf484a413ce4c628835b9a331e9 Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Wed, 10 May 2023 10:39:55 +0800 Subject: [PATCH 10/16] [feature][plugin][sybase] datax support sybase plugins. #1780 update package position --- sybasewriter/src/main/{java => }/assembly/package.xml | 1 + 1 file changed, 1 insertion(+) rename sybasewriter/src/main/{java => }/assembly/package.xml (99%) diff --git a/sybasewriter/src/main/java/assembly/package.xml b/sybasewriter/src/main/assembly/package.xml similarity index 99% rename from sybasewriter/src/main/java/assembly/package.xml rename to sybasewriter/src/main/assembly/package.xml index 511c5627..15500d3d 100755 --- a/sybasewriter/src/main/java/assembly/package.xml +++ b/sybasewriter/src/main/assembly/package.xml @@ -32,4 +32,5 @@ runtime + From ec2402b546e21907a8353bbc199d26b92f5babad Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Wed, 10 May 2023 10:40:19 +0800 Subject: [PATCH 11/16] [feature][plugin][sybase] datax support sybase plugins. #1780 add sybaseWriter code --- .../{reader => writer}/sybasewriter/SybaseWriter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename sybasewriter/src/main/java/com/alibaba/datax/plugin/{reader => writer}/sybasewriter/SybaseWriter.java (93%) diff --git a/sybasewriter/src/main/java/com/alibaba/datax/plugin/reader/sybasewriter/SybaseWriter.java b/sybasewriter/src/main/java/com/alibaba/datax/plugin/writer/sybasewriter/SybaseWriter.java similarity index 93% rename from sybasewriter/src/main/java/com/alibaba/datax/plugin/reader/sybasewriter/SybaseWriter.java rename to sybasewriter/src/main/java/com/alibaba/datax/plugin/writer/sybasewriter/SybaseWriter.java index 220e6233..51b90d66 100755 --- a/sybasewriter/src/main/java/com/alibaba/datax/plugin/reader/sybasewriter/SybaseWriter.java +++ b/sybasewriter/src/main/java/com/alibaba/datax/plugin/writer/sybasewriter/SybaseWriter.java @@ -1,4 +1,4 @@ -package com.alibaba.datax.plugin.reader.sybasewriter; +package com.alibaba.datax.plugin.writer.sybasewriter; import com.alibaba.datax.common.plugin.RecordReceiver; import com.alibaba.datax.common.spi.Writer; @@ -6,13 +6,14 @@ import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.plugin.rdbms.util.DataBaseType; import com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter; import com.alibaba.datax.plugin.rdbms.writer.Key; +import com.alibaba.datax.plugin.rdbms.util.DataBaseType; + import java.util.List; public class SybaseWriter extends Writer { - private static final DataBaseType DATABASE_TYPE = DataBaseType.SYBASE; - + private static final DataBaseType DATABASE_TYPE = DataBaseType.Sybase; public static class Job extends Writer.Job { private Configuration originalConfig = null; private CommonRdbmsWriter.Job commonRdbmsWriterJob; @@ -72,7 +73,6 @@ public class SybaseWriter extends Writer { this.commonRdbmsWriterTask.prepare(this.writerSliceConfig); } - //TODO 改用连接池，确保每次获取的连接都是可用的（注意：连接可能需要每次都初始化其 session） public void startWrite(RecordReceiver recordReceiver) { this.commonRdbmsWriterTask.startWrite(recordReceiver, this.writerSliceConfig, super.getTaskPluginCollector()); From a10342b7d102762958f94eec0c2b0d423753f393 Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Wed, 10 May 2023 10:43:34 +0800 Subject: [PATCH 12/16] [feature][plugin][sybase] datax support sybase plugins. #1780 add module into pom.xml --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index 957c60ee..52f80372 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,8 @@ loghubreader datahubreader starrocksreader + sybasereader + mysqlwriter @@ -123,6 +125,7 @@ doriswriter selectdbwriter adbmysqlwriter + sybasewriter plugin-rdbms-util From d6dbe17abd55fb2aefd1b6adfb01c25569ef4008 Mon Sep 17 00:00:00 2001 From: xiaopengm <602012854@qq.com> Date: Wed, 10 May 2023 11:29:36 +0800 Subject: [PATCH 13/16] [feature][plugin][sybase] datax support sybase plugins. #1780 modify DataBaseType --- .../java/com/alibaba/datax/plugin/rdbms/util/DataBaseType.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/DataBaseType.java b/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/DataBaseType.java index e33ea1a6..921241c6 100755 --- a/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/DataBaseType.java +++ b/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/DataBaseType.java @@ -134,6 +134,8 @@ public enum DataBaseType { result = jdbc + "?" + suffix; } break; + case Sybase: + break; default: throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type."); } From 4095a2a95484e5f4a24dd5968d772569ecebd52b Mon Sep 17 00:00:00 2001 From: mengxiaopeng <602012854@qq.com> Date: Wed, 5 Jul 2023 15:25:42 +0800 Subject: [PATCH 14/16] [feature][plugin][sybase] datax support sybase plugins. #1780 add unit testcase --- sybasereader/pom.xml | 6 ++ .../sybasereader/SybaseDatabaseUnitTest.java | 55 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 sybasereader/src/test/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseDatabaseUnitTest.java diff --git a/sybasereader/pom.xml b/sybasereader/pom.xml index 1c826fb4..cc3a3840 100644 --- a/sybasereader/pom.xml +++ b/sybasereader/pom.xml @@ -65,6 +65,12 @@ ${project.basedir}/libs/jconn4-16.0.jar + + junit + junit + 4.13.2 + test + diff --git a/sybasereader/src/test/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseDatabaseUnitTest.java b/sybasereader/src/test/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseDatabaseUnitTest.java new file mode 100644 index 00000000..f77caccd --- /dev/null +++ b/sybasereader/src/test/java/com/alibaba/datax/plugin/reader/sybasereader/SybaseDatabaseUnitTest.java @@ -0,0 +1,55 @@ +package com.alibaba.datax.plugin.reader.sybasereader; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.junit.Assert.assertEquals; + +public class SybaseDatabaseUnitTest { + private Connection connection; + + @Before + public void setUp() { + // 连接到 Sybase 数据库 + String jdbcUrl = "jdbc:sybase:Tds:192.172.172.80:1680/database"; + String username = "admin"; + String password = "admin123"; + + try { + connection = DriverManager.getConnection(jdbcUrl, username, password); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @After + public void tearDown() { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + @Test + public void testDatabaseQuery() throws SQLException { + String query = "SELECT COUNT(*) FROM your_table"; + int expectedRowCount = 10; // 假设期望返回的行数是 10 + + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(query); + resultSet.next(); + int rowCount = resultSet.getInt(1); + + assertEquals(expectedRowCount, rowCount); + } +} From 0280de66d32024de66eaa746c7efe0e9c3932cd8 Mon Sep 17 00:00:00 2001 From: mengxiaopeng Date: Wed, 27 Mar 2024 10:30:39 +0800 Subject: [PATCH 15/16] =?UTF-8?q?[+]=20Pom.xml=E5=A2=9E=E5=8A=A0SybaseWrit?= =?UTF-8?q?er=E5=92=8CneoWriter=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ad99d4ed..eeb4bfaf 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,8 @@ doriswriter selectdbwriter adbmysqlwriter - + sybasewriter + neo4jwriter plugin-rdbms-util plugin-unstructured-storage-util From 6107e01c9f8d2f07f5da3e73fc9d3f2e45c50708 Mon Sep 17 00:00:00 2001 From: mengxiaopeng Date: Wed, 27 Mar 2024 10:37:39 +0800 Subject: [PATCH 16/16] =?UTF-8?q?[+]=20package.xml=20=E5=A2=9E=E5=8A=A0syb?= =?UTF-8?q?ase=E5=8C=85=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/package.xml b/package.xml index 5ea8bd22..e51c11e1 100644 --- a/package.xml +++ b/package.xml @@ -243,6 +243,13 @@ datax + + sybasereader/target/datax/ + + **/*.* + + datax + @@ -518,5 +525,12 @@ datax + + sybasewriter/target/datax/ + + **/*.* + + datax +