diff --git a/rdb2graph/README.md b/rdb2graph/README.md new file mode 100644 index 00000000..b0cedf64 --- /dev/null +++ b/rdb2graph/README.md @@ -0,0 +1,3 @@ +- 关系/图转换插件，该插件属于数据同构同步，模型一致，属性名称一致，优点是自动化高，不用多大干预，适合生成中间数据， + 特别是，关系数据库系统与图数据库系统不在同一网段，网络不稳定，同步尽量简单显得很重要 +- schema转换 4.3+ diff --git a/rdb2graph/mysqlreader4graph/doc/mysqlreader.md b/rdb2graph/mysqlreader4graph/doc/mysqlreader.md new file mode 100644 index 00000000..3ae52afb --- /dev/null +++ b/rdb2graph/mysqlreader4graph/doc/mysqlreader.md @@ -0,0 +1,368 @@ + +# MysqlReader 插件文档 + + +___ + + + +## 1 快速介绍 + +MysqlReader插件实现了从Mysql读取数据。在底层实现上，MysqlReader通过JDBC连接远程Mysql数据库，并执行相应的sql语句将数据从mysql库中SELECT出来。 + +**不同于其他关系型数据库，MysqlReader不支持FetchSize.** + +## 2 实现原理 + +简而言之，MysqlReader通过JDBC连接器连接到远程的Mysql数据库，并根据用户配置的信息生成查询SELECT SQL语句，然后发送到远程Mysql数据库，并将该SQL执行返回结果使用DataX自定义的数据类型拼装为抽象的数据集，并传递给下游Writer处理。 + +对于用户配置Table、Column、Where的信息，MysqlReader将其拼接为SQL语句发送到Mysql数据库；对于用户配置querySql信息，MysqlReader直接将其发送到Mysql数据库。 + + +## 3 功能说明 + +### 3.1 配置样例 + +* 配置一个从Mysql数据库同步抽取数据到本地的作业: + +``` +{ + "job": { + "setting": { + "speed": { + "channel": 3 + }, + "errorLimit": { + "record": 0, + "percentage": 0.02 + } + }, + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "username": "root", + "password": "root", + "column": [ + "id", + "name" + ], + "splitPk": "db_id", + "connection": [ + { + "table": [ + "table" + ], + "jdbcUrl": [ + "jdbc:mysql://127.0.0.1:3306/database" + ] + } + ] + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "print":true + } + } + } + ] + } +} + +``` + +* 配置一个自定义SQL的数据库同步任务到本地内容的作业： + +``` +{ + "job": { + "setting": { + "speed": { + "channel":1 + } + }, + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "username": "root", + "password": "root", + "connection": [ + { + "querySql": [ + "select db_id,on_line_flag from db_info where db_id < 10;" + ], + "jdbcUrl": [ + "jdbc:mysql://bad_ip:3306/database", + "jdbc:mysql://127.0.0.1:bad_port/database", + "jdbc:mysql://127.0.0.1:3306/database" + ] + } + ] + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "print": false, + "encoding": "UTF-8" + } + } + } + ] + } +} +``` + + +### 3.2 参数说明 + +* **jdbcUrl** + + * 描述：描述的是到对端数据库的JDBC连接信息，使用JSON的数组描述，并支持一个库填写多个连接地址。之所以使用JSON数组描述连接信息，是因为阿里集团内部支持多个IP探测，如果配置了多个，MysqlReader可以依次探测ip的可连接性，直到选择一个合法的IP。如果全部连接失败，MysqlReader报错。 注意，jdbcUrl必须包含在connection配置单元中。对于阿里集团外部使用情况，JSON数组填写一个JDBC连接即可。 + + jdbcUrl按照Mysql官方规范，并可以填写连接附件控制信息。具体请参看[Mysql官方文档](http://dev.mysql.com/doc/connector-j/en/connector-j-reference-configuration-properties.html)。 + + * 必选：是
+ + * 默认值：无
+ +* **username** + + * 描述：数据源的用户名
+ + * 必选：是
+ + * 默认值：无
+ +* **password** + + * 描述：数据源指定用户名的密码
+ + * 必选：是
+ + * 默认值：无
+ +* **table** + + * 描述：所选取的需要同步的表。使用JSON的数组描述，因此支持多张表同时抽取。当配置为多张表时，用户自己需保证多张表是同一schema结构，MysqlReader不予检查表是否同一逻辑表。注意，table必须包含在connection配置单元中。
+ + * 必选：是
+ + * 默认值：无
+ +* **column** + + * 描述：所配置的表中需要同步的列名集合，使用JSON的数组描述字段信息。用户使用\*代表默认使用所有列配置，例如['\*']。 + + 支持列裁剪，即列可以挑选部分列进行导出。 + + 支持列换序，即列可以不按照表schema信息进行导出。 + + 支持常量配置，用户需要按照Mysql SQL语法格式: + ["id", "\`table\`", "1", "'bazhen.csy'", "null", "to_char(a + 1)", "2.3" , "true"] + id为普通列名，\`table\`为包含保留在的列名，1为整形数字常量，'bazhen.csy'为字符串常量，null为空指针，to_char(a + 1)为表达式，2.3为浮点数，true为布尔值。 + + * 必选：是
+ + * 默认值：无
+ +* **splitPk** + + * 描述：MysqlReader进行数据抽取时，如果指定splitPk，表示用户希望使用splitPk代表的字段进行数据分片，DataX因此会启动并发任务进行数据同步，这样可以大大提供数据同步的效能。 + + 推荐splitPk用户使用表主键，因为表主键通常情况下比较均匀，因此切分出来的分片也不容易出现数据热点。 + +  目前splitPk仅支持整形数据切分，`不支持浮点、字符串、日期等其他类型`。如果用户指定其他非支持类型，MysqlReader将报错！ + + 如果splitPk不填写，包括不提供splitPk或者splitPk值为空，DataX视作使用单通道同步该表数据。 + + * 必选：否
+ + * 默认值：空
+ +* **where** + + * 描述：筛选条件，MysqlReader根据指定的column、table、where条件拼接SQL，并根据这个SQL进行数据抽取。在实际业务场景中，往往会选择当天的数据进行同步，可以将where条件指定为gmt_create > $bizdate 。注意：不可以将where条件指定为limit 10，limit不是SQL的合法where子句。
+ + where条件可以有效地进行业务增量同步。如果不填写where语句，包括不提供where的key或者value，DataX均视作同步全量数据。 + + * 必选：否
+ + * 默认值：无
+ +* **querySql** + + * 描述：在有些业务场景下，where这一配置项不足以描述所筛选的条件，用户可以通过该配置型来自定义筛选SQL。当用户配置了这一项之后，DataX系统就会忽略table，column这些配置型，直接使用这个配置项的内容对数据进行筛选，例如需要进行多表join后同步数据，使用select a,b from table_a join table_b on table_a.id = table_b.id
+ + `当用户配置querySql时，MysqlReader直接忽略table、column、where条件的配置`，querySql优先级大于table、column、where选项。 + + * 必选：否
+ + * 默认值：无
+ + +### 3.3 类型转换 + +目前MysqlReader支持大部分Mysql类型，但也存在部分个别类型没有支持的情况，请注意检查你的类型。 + +下面列出MysqlReader针对Mysql类型转换列表: + + +| DataX 内部类型| Mysql 数据类型 | +| -------- | ----- | +| Long |int, tinyint, smallint, mediumint, int, bigint| +| Double |float, double, decimal| +| String |varchar, char, tinytext, text, mediumtext, longtext, year | +| Date |date, datetime, timestamp, time | +| Boolean |bit, bool | +| Bytes |tinyblob, mediumblob, blob, longblob, varbinary | + + + +请注意: + +* `除上述罗列字段类型外，其他类型均不支持`。 +* `tinyint(1) DataX视作为整形`。 +* `year DataX视作为字符串类型` +* `bit DataX属于未定义行为`。 + +## 4 性能报告 + +### 4.1 环境准备 + +#### 4.1.1 数据特征 +建表语句： + + CREATE TABLE `tc_biz_vertical_test_0000` ( + `biz_order_id` bigint(20) NOT NULL COMMENT 'id', + `key_value` varchar(4000) NOT NULL COMMENT 'Key-value的内容', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `attribute_cc` int(11) DEFAULT NULL COMMENT '防止并发修改的标志', + `value_type` int(11) NOT NULL DEFAULT '0' COMMENT '类型', + `buyer_id` bigint(20) DEFAULT NULL COMMENT 'buyerid', + `seller_id` bigint(20) DEFAULT NULL COMMENT 'seller_id', + PRIMARY KEY (`biz_order_id`,`value_type`), + KEY `idx_biz_vertical_gmtmodified` (`gmt_modified`) + ) ENGINE=InnoDB DEFAULT CHARSET=gbk COMMENT='tc_biz_vertical' + + +单行记录类似于： + + biz_order_id: 888888888 + key_value: ;orderIds:20148888888,2014888888813800; + gmt_create: 2011-09-24 11:07:20 + gmt_modified: 2011-10-24 17:56:34 + attribute_cc: 1 + value_type: 3 + buyer_id: 8888888 + seller_id: 1 + +#### 4.1.2 机器参数 + +* 执行DataX的机器参数为: + 1. cpu: 24核 Intel(R) Xeon(R) CPU E5-2630 0 @ 2.30GHz + 2. mem: 48GB + 3. net: 千兆双网卡 + 4. disc: DataX 数据不落磁盘，不统计此项 + +* Mysql数据库机器参数为: + 1. cpu: 32核 Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz + 2. mem: 256GB + 3. net: 千兆双网卡 + 4. disc: BTWL419303E2800RGN INTEL SSDSC2BB800G4 D2010370 + +#### 4.1.3 DataX jvm 参数 + + -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + + +### 4.2 测试报告 + +#### 4.2.1 单表测试报告 + + +| 通道数| 是否按照主键切分| DataX速度(Rec/s)|DataX流量(MB/s)| DataX机器网卡进入流量(MB/s)|DataX机器运行负载|DB网卡流出流量(MB/s)|DB运行负载| +|--------|--------| --------|--------|--------|--------|--------|--------| +|1| 否 | 183185 | 18.11 | 29| 0.6 | 31| 0.6 | +|1| 是 | 183185 | 18.11 | 29| 0.6 | 31| 0.6 | +|4| 否 | 183185 | 18.11 | 29| 0.6 | 31| 0.6 | +|4| 是 | 329733 | 32.60 | 58| 0.8 | 60| 0.76 | +|8| 否 | 183185 | 18.11 | 29| 0.6 | 31| 0.6 | +|8| 是 | 549556 | 54.33 | 115| 1.46 | 120| 0.78 | + +说明： + +1. 这里的单表，主键类型为 bigint(20),范围为：190247559466810-570722244711460，从主键范围划分看，数据分布均匀。 +2. 对单表如果没有安装主键切分，那么配置通道个数不会提升速度，效果与1个通道一样。 + + +#### 4.2.2 分表测试报告(2个分库，每个分库16张分表，共计32张分表) + + +| 通道数| DataX速度(Rec/s)|DataX流量(MB/s)| DataX机器网卡进入流量(MB/s)|DataX机器运行负载|DB网卡流出流量(MB/s)|DB运行负载| +|--------| --------|--------|--------|--------|--------|--------| +|1| 202241 | 20.06 | 31.5| 1.0 | 32 | 1.1 | +|4| 726358 | 72.04 | 123.9 | 3.1 | 132 | 3.6 | +|8|1074405 | 106.56| 197 | 5.5 | 205| 5.1| +|16| 1227892 | 121.79 | 229.2 | 8.1 | 233 | 7.3 | + +## 5 约束限制 + +### 5.1 主备同步数据恢复问题 + +主备同步问题指Mysql使用主从灾备，备库从主库不间断通过binlog恢复数据。由于主备数据同步存在一定的时间差，特别在于某些特定情况，例如网络延迟等问题，导致备库同步恢复的数据与主库有较大差别，导致从备库同步的数据不是一份当前时间的完整镜像。 + +针对这个问题，我们提供了preSql功能，该功能待补充。 + +### 5.2 一致性约束 + +Mysql在数据存储划分中属于RDBMS系统，对外可以提供强一致性数据查询接口。例如当一次同步任务启动运行过程中，当该库存在其他数据写入方写入数据时，MysqlReader完全不会获取到写入更新数据，这是由于数据库本身的快照特性决定的。关于数据库快照特性，请参看[MVCC Wikipedia](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) + +上述是在MysqlReader单线程模型下数据同步一致性的特性，由于MysqlReader可以根据用户配置信息使用了并发数据抽取，因此不能严格保证数据一致性：当MysqlReader根据splitPk进行数据切分后，会先后启动多个并发任务完成数据同步。由于多个并发任务相互之间不属于同一个读事务，同时多个并发任务存在时间间隔。因此这份数据并不是`完整的`、`一致的`数据快照信息。 + +针对多线程的一致性快照需求，在技术上目前无法实现，只能从工程角度解决，工程化的方式存在取舍，我们提供几个解决思路给用户，用户可以自行选择： + +1. 使用单线程同步，即不再进行数据切片。缺点是速度比较慢，但是能够很好保证一致性。 + +2. 关闭其他数据写入方，保证当前数据为静态数据，例如，锁表、关闭备库同步等等。缺点是可能影响在线业务。 + +### 5.3 数据库编码问题 + +Mysql本身的编码设置非常灵活，包括指定编码到库、表、字段级别，甚至可以均不同编码。优先级从高到低为字段、表、库、实例。我们不推荐数据库用户设置如此混乱的编码，最好在库级别就统一到UTF-8。 + +MysqlReader底层使用JDBC进行数据抽取，JDBC天然适配各类编码，并在底层进行了编码转换。因此MysqlReader不需用户指定编码，可以自动获取编码并转码。 + +对于Mysql底层写入编码和其设定的编码不一致的混乱情况，MysqlReader对此无法识别，对此也无法提供解决方案，对于这类情况，`导出有可能为乱码`。 + +### 5.4 增量数据同步 + +MysqlReader使用JDBC SELECT语句完成数据抽取工作，因此可以使用SELECT...WHERE...进行增量数据抽取，方式有多种： + +* 数据库在线应用写入数据库时，填充modify字段为更改时间戳，包括新增、更新、删除(逻辑删)。对于这类应用，MysqlReader只需要WHERE条件跟上一同步阶段时间戳即可。 +* 对于新增流水型数据，MysqlReader可以WHERE条件后跟上一阶段最大自增ID即可。 + +对于业务上无字段区分新增、修改数据情况，MysqlReader也无法进行增量数据同步，只能同步全量数据。 + +### 5.5 Sql安全性 + +MysqlReader提供querySql语句交给用户自己实现SELECT抽取语句，MysqlReader本身对querySql不做任何安全性校验。这块交由DataX用户方自己保证。 + +## 6 FAQ + +*** + +**Q: MysqlReader同步报错，报错信息为XXX** + + A: 网络或者权限问题，请使用mysql命令行测试： + + mysql -u -p -h -D -e "select * from <表名>" + +如果上述命令也报错，那可以证实是环境问题，请联系你的DBA。 + + diff --git a/rdb2graph/mysqlreader4graph/pom.xml b/rdb2graph/mysqlreader4graph/pom.xml new file mode 100644 index 00000000..a6238e30 --- /dev/null +++ b/rdb2graph/mysqlreader4graph/pom.xml @@ -0,0 +1,84 @@ + + + 4.0.0 + + com.leehom.arch.datax.plugin + rdb2graph-parent + ${revision} + + mysqlreader4graph + jar + + + + com.alibaba.datax + datax-common + + + slf4j-log4j12 + org.slf4j + + + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + + mysql + mysql-connector-java + ${mysql.driver.version} + + + + com.leehom.arch.datax.plugin + rdb2graph-rdbms-util + ${revision} + + + com.leehom.arch.datax.plugin + rdb2graph-common + ${revision} + + + + + + + + maven-compiler-plugin + + ${java.version} + ${java.version} + ${project-sourceEncoding} + + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + diff --git a/rdb2graph/mysqlreader4graph/src/main/assembly/package.xml b/rdb2graph/mysqlreader4graph/src/main/assembly/package.xml new file mode 100644 index 00000000..1d81c407 --- /dev/null +++ b/rdb2graph/mysqlreader4graph/src/main/assembly/package.xml @@ -0,0 +1,35 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/reader/mysqlreader4graph + + + target/ + + mysqlreader4graph-1.0.0-SNAPSHOT.jar + + plugin/reader/mysqlreader4graph + + + + + + false + plugin/reader/mysqlreader4graph/libs + runtime + + + diff --git a/rdb2graph/mysqlreader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/mysqlreader/MysqlReader4Graph.java b/rdb2graph/mysqlreader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/mysqlreader/MysqlReader4Graph.java new file mode 100644 index 00000000..5e33c212 --- /dev/null +++ b/rdb2graph/mysqlreader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/mysqlreader/MysqlReader4Graph.java @@ -0,0 +1,110 @@ +package com.leehom.arch.datax.plugin.rdb2graph.reader.mysqlreader; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.common.spi.Reader; +import com.alibaba.datax.common.util.Configuration; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.CommonRdbms2GraphReader; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Constant; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DataBaseType; + +/** + * @类名: MysqlReader4Graph + * @说明: mysql读入 + * 1.表数据同步，datax table模式，schema表写入datax的配置 + 2.表关系同步，按表连接图分组，一对关系一组，表连接关系生成querySql，写入datax的querySql模式配置 + * + * @author leehom + * @Date 2022年4月30日 上午12:38:34 + * 修改记录： + * + * @see + */ +public class MysqlReader4Graph extends Reader { + + private static final DataBaseType DATABASE_TYPE = DataBaseType.MySql; + + public static class Job extends Reader.Job { + private static final Logger LOG = LoggerFactory + .getLogger(Job.class); + + private Configuration originalConfig = null; + private CommonRdbms2GraphReader.Job commonRdbmsReaderJob; + + @Override + public void init() { + this.originalConfig = super.getPluginJobConf(); + + Integer userConfigedFetchSize = this.originalConfig.getInt(Constant.FETCH_SIZE); + if (userConfigedFetchSize != null) { + LOG.warn("对 mysqlreader 不需要配置 fetchSize, mysqlreader 将会忽略这项配置. 如果您不想再看到此警告,请去除fetchSize 配置."); + } + + this.originalConfig.set(Constant.FETCH_SIZE, Integer.MIN_VALUE); + + this.commonRdbmsReaderJob = new CommonRdbms2GraphReader.Job(DATABASE_TYPE); + this.commonRdbmsReaderJob.init(this.originalConfig); + } + + @Override + public void preCheck(){ + init(); + this.commonRdbmsReaderJob.preCheck(this.originalConfig,DATABASE_TYPE); + + } + + @Override + public List split(int adviceNumber) { + return this.commonRdbmsReaderJob.split(this.originalConfig, adviceNumber); + } + + @Override + public void post() { + this.commonRdbmsReaderJob.post(this.originalConfig); + } + + @Override + public void destroy() { + this.commonRdbmsReaderJob.destroy(this.originalConfig); + } + + } + + public static class Task extends Reader.Task { + + private Configuration readerSliceConfig; + private CommonRdbms2GraphReader.Task commonRdbmsReaderTask; + + @Override + public void init() { + this.readerSliceConfig = super.getPluginJobConf(); + this.commonRdbmsReaderTask = new CommonRdbms2GraphReader.Task(DATABASE_TYPE,super.getTaskGroupId(), super.getTaskId()); + this.commonRdbmsReaderTask.init(this.readerSliceConfig); + + } + + @Override + public void startRead(RecordSender recordSender) { + int fetchSize = this.readerSliceConfig.getInt(Constant.FETCH_SIZE); + + this.commonRdbmsReaderTask.startRead(this.readerSliceConfig, recordSender, + super.getTaskPluginCollector(), fetchSize); + } + + @Override + public void post() { + this.commonRdbmsReaderTask.post(this.readerSliceConfig); + } + + @Override + public void destroy() { + this.commonRdbmsReaderTask.destroy(this.readerSliceConfig); + } + + } + +} diff --git a/rdb2graph/mysqlreader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/mysqlreader/MysqlReaderErrorCode.java b/rdb2graph/mysqlreader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/mysqlreader/MysqlReaderErrorCode.java new file mode 100644 index 00000000..8cda39a9 --- /dev/null +++ b/rdb2graph/mysqlreader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/mysqlreader/MysqlReaderErrorCode.java @@ -0,0 +1,31 @@ +package com.leehom.arch.datax.plugin.rdb2graph.reader.mysqlreader; + +import com.alibaba.datax.common.spi.ErrorCode; + +public enum MysqlReaderErrorCode implements ErrorCode { + ; + + private final String code; + private final String description; + + private MysqlReaderErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s]. ", this.code, + this.description); + } +} diff --git a/rdb2graph/mysqlreader4graph/src/main/resources/plugin.json b/rdb2graph/mysqlreader4graph/src/main/resources/plugin.json new file mode 100644 index 00000000..3844d545 --- /dev/null +++ b/rdb2graph/mysqlreader4graph/src/main/resources/plugin.json @@ -0,0 +1,6 @@ +{ + "name": "mysqlreader4graph", + "class": "com.leehom.arch.datax.plugin.rdb2graph.reader.mysqlreader.MysqlReader4Graph", + "description": "", + "developer": "leehom" +} \ No newline at end of file diff --git a/rdb2graph/mysqlreader4graph/src/main/resources/plugin_job_template.json b/rdb2graph/mysqlreader4graph/src/main/resources/plugin_job_template.json new file mode 100644 index 00000000..c330e9fb --- /dev/null +++ b/rdb2graph/mysqlreader4graph/src/main/resources/plugin_job_template.json @@ -0,0 +1,17 @@ +{ + "name": "mysqlreader4graph", + "parameter": { + "username": "", + "password": "", + "phase": "", + "schemaUri": "", + "column": [], + "connection": [ + { + "jdbcUrl": [], + "table": [] + } + ], + "where": "" + } +} \ No newline at end of file diff --git a/rdb2graph/neo4jwriter/README.md b/rdb2graph/neo4jwriter/README.md new file mode 100644 index 00000000..fd40910d --- /dev/null +++ b/rdb2graph/neo4jwriter/README.md @@ -0,0 +1,4 @@ + + + + diff --git a/rdb2graph/neo4jwriter/pom.xml b/rdb2graph/neo4jwriter/pom.xml new file mode 100644 index 00000000..d19df925 --- /dev/null +++ b/rdb2graph/neo4jwriter/pom.xml @@ -0,0 +1,84 @@ + + + + com.leehom.arch.datax.plugin + rdb2graph-parent + ${revision} + + 4.0.0 + + neo4jwriter + + + + com.alibaba.datax + datax-common + ${datax.version} + + + slf4j-log4j12 + org.slf4j + + + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + com.alibaba + fastjson + + + junit + junit + test + + + com.leehom.arch.datax.plugin + rdb2graph-scanner + 1.0.0-SNAPSHOT + + + com.leehom.arch.datax.plugin + rdb2graph-common + 1.0.0-SNAPSHOT + + + + + + + maven-compiler-plugin + + ${java.version} + ${java.version} + ${project-sourceEncoding} + + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + diff --git a/rdb2graph/neo4jwriter/src/main/assembly/package.xml b/rdb2graph/neo4jwriter/src/main/assembly/package.xml new file mode 100644 index 00000000..2d0cb951 --- /dev/null +++ b/rdb2graph/neo4jwriter/src/main/assembly/package.xml @@ -0,0 +1,34 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + + plugin/writer/neo4jwriter + + + target/ + + neo4jwriter-1.0.0-SNAPSHOT.jar + + plugin/writer/neo4jwriter + + + + + + false + plugin/writer/neo4jwriter/libs + runtime + + + diff --git a/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Key.java b/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Key.java new file mode 100644 index 00000000..ae397da2 --- /dev/null +++ b/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Key.java @@ -0,0 +1,121 @@ +package com.leehom.arch.datax.plugin.rdb2graph.writer.neo4jwriter; + +import com.alibaba.datax.common.util.Configuration; +import org.apache.commons.lang3.StringUtils; + +import java.util.HashMap; +import java.util.Map; + +/** + * @类名: Key + * @说明: 配置项 + * + * @author leehom + * @Date 2022年4月27日 下午2:49:13 + * 修改记录： + * + * @see + */ +public final class Key { + + public static String database(Configuration conf) { + return conf.getString("database", ""); + } + + public static String schemaUri(Configuration conf) { + return conf.getString("schemaUri", ""); + } + + public static String uri(Configuration conf) { + return conf.getString("uri", ""); + } + + public static String userName(Configuration conf) { + return conf.getString("username", ""); + } + + public static String password(Configuration conf) { + return conf.getString("password", ""); + } + + public static int batchSize(Configuration conf) { + return conf.getInt("batchSize", 1000); + } + + public static int getTrySize(Configuration conf) { + return conf.getInt("trySize", 30); + } + + public static int getTimeout(Configuration conf) { + return conf.getInt("timeout", 600000); + } + + public static boolean isCleanup(Configuration conf) { + return conf.getBool("cleanup", false); + } + + public static boolean isDiscovery(Configuration conf) { + return conf.getBool("discovery", false); + } + + public static boolean isCompression(Configuration conf) { + return conf.getBool("compression", true); + } + + public static boolean isMultiThread(Configuration conf) { + return conf.getBool("multiThread", true); + } + + public static String getIndexName(Configuration conf) { + return conf.getNecessaryValue("index", Neo4jWriterErrorCode.BAD_CONFIG_VALUE); + } + + public static String getTypeName(Configuration conf) { + String indexType = conf.getString("indexType"); + if(StringUtils.isBlank(indexType)){ + indexType = conf.getString("type", getIndexName(conf)); + } + return indexType; + } + + + public static boolean isIgnoreWriteError(Configuration conf) { + return conf.getBool("ignoreWriteError", false); + } + + public static boolean isIgnoreParseError(Configuration conf) { + return conf.getBool("ignoreParseError", true); + } + + + public static boolean isHighSpeedMode(Configuration conf) { + if ("highspeed".equals(conf.getString("mode", ""))) { + return true; + } + return false; + } + + public static String getAlias(Configuration conf) { + return conf.getString("alias", ""); + } + + public static boolean isNeedCleanAlias(Configuration conf) { + String mode = conf.getString("aliasMode", "append"); + if ("exclusive".equals(mode)) { + return true; + } + return false; + } + + public static Map getSettings(Configuration conf) { + return conf.getMap("settings", new HashMap()); + } + + public static String getSplitter(Configuration conf) { + return conf.getString("splitter", "-,-"); + } + + public static boolean getDynamic(Configuration conf) { + return conf.getBool("dynamic", false); + } +} diff --git a/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Neo4jWriter.java b/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Neo4jWriter.java new file mode 100644 index 00000000..edf6aef5 --- /dev/null +++ b/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Neo4jWriter.java @@ -0,0 +1,424 @@ +package com.leehom.arch.datax.plugin.rdb2graph.writer.neo4jwriter; + +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.sql.JDBCType; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.reflect.MethodUtils; +import org.neo4j.driver.AuthTokens; +import org.neo4j.driver.Driver; +import org.neo4j.driver.GraphDatabase; +import org.neo4j.driver.Query; +import org.neo4j.driver.exceptions.Neo4jException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Column.Type; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.spi.Writer; +import com.alibaba.datax.common.util.Configuration; +import com.leehom.arch.datax.plugin.rdb2graph.common.ByteAndStreamUtils; +import com.leehom.arch.datax.plugin.rdb2graph.common.ResourceLoaderUtil; +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.Serializer; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.config.ScannerSerializerConfig; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jDao; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jQueryPattern; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.ParamsUtils; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKField; + +/** + * @类名: Neo4jWriter + * @说明: neo4j写入，支持两个阶段 + * + * + * @author leehom + * @Date 2022年4月21日 下午11:14:32 + * 修改记录： + * + * TODO + * 1. 支持写入模式，@WriteMode replace/update + * + * @see + */ +public class Neo4jWriter extends Writer { + + public static class Job extends Writer.Job { + + private static final Logger log = LoggerFactory.getLogger(Job.class); + + private Configuration conf = null; + + // 初始化环境和配置，为客户端准备环境 + @Override + public void init() { + this.conf = super.getPluginJobConf(); + } + + // 准备 + // 这里可做schema转换 + @Override + public void prepare() { + // + } + + // 目标库只有一个，分片相当于并行写入，克隆即可 + @Override + public List split(int mandatoryNumber) { + List configurations = new ArrayList(mandatoryNumber); + for (int i = 0; i < mandatoryNumber; i++) { + configurations.add(conf); + } + return configurations; + } + + // 后处理 + @Override + public void post() { + // nothing + } + + // 释放资源 + @Override + public void destroy() { + // nothing + } + } + + public static class Task extends Writer.Task { + + private static final Logger log = LoggerFactory.getLogger(Job.class); + + private Configuration conf; + private Neo4jDao client = null; + private int batchSize; // 支持批量 + private DbSchema rdbSchema; // + private WriteMode writeMode; // 写入模式， + private Serializer ser; // + + @Override + public void init() { + // + this.conf = super.getPluginJobConf(); + // 初始化 Neo4jDao + String uri = Key.uri(conf); + String un = Key.userName(conf); + String pw = Key.password(conf); + String db = Key.database(conf); + Driver driver = GraphDatabase.driver(uri, AuthTokens.basic(un, pw)); + client = new Neo4jDao(); + client.setDriver(driver); + client.setDatabase(db); + // 初始化数据库schema序列化器 + ser = ScannerSerializerConfig.rdbSchemaXmlSerializer(); + // 写入批量 + batchSize = Key.batchSize(conf); + } + + @Override + public void prepare() { + init(); // 作业容器并没有调用init方法 + // 载入关系数据库schema + String schemaUri = Key.schemaUri(conf); + InputStream is; + try { + is = ResourceLoaderUtil.getResourceStream(schemaUri); + byte[] bytes = ByteAndStreamUtils.StreamToBytes(is); + rdbSchema = (DbSchema)ser.Unmarshal(bytes); + } catch (Exception e) { + DataXException.asDataXException(Neo4jWriterErrorCode.ERROR_LOAD_RDBSCHEMA, e); + } + + } + + // 写入, 两个场景 + // 1. 表记录 TableRecord 2. 关系记录 RelRecord + @Override + public void startWrite(RecordReceiver recordReceiver) { + List writerBuffer = new ArrayList(this.batchSize); + Record record = null; + // 交换器(Exchanger)接收到TerminateRecord返回null + while ((record = recordReceiver.getFromReader()) != null) { + writerBuffer.add(record); + if (writerBuffer.size() >= this.batchSize) { + this.doWrite(writerBuffer); + writerBuffer.clear(); + } + } + if (!writerBuffer.isEmpty()) { + this.doWrite(writerBuffer); + writerBuffer.clear(); + } + + } + + // 写入分流 节点 / 关系 + private void doWrite(List writerBuffer) { + Record record = writerBuffer.get(0); + try { + if ("TableRecord".equals(record.getClass().getSimpleName())) { + doBatchWriteNode(writerBuffer); + return; + } + if ("RelRecord".equals(record.getClass().getSimpleName())) { + doBatchWriteRel(writerBuffer); + return; + } + // 系统sleep + try { + Thread.sleep(300); + } catch (InterruptedException e) { + + } + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + // 理论上不会发送，只记录 + log.error(e.getMessage()); + + } catch (DataXException e) { + // 字段类型不合法，记录，其他抛出 + log.error(e.getMessage()); + + } catch (Neo4jException e) { + // neo4j异常 + throw DataXException.asDataXException(Neo4jWriterErrorCode.WRONG_NEO4j_CLIENT, + Neo4jWriterErrorCode.WRONG_NEO4j_CLIENT.getDescription(), e); + } + + } + + // 批量写入 + private void doBatchWriteNode(final List writerBuffer) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + List queries = new ArrayList<>(); + for (Record record : writerBuffer) { + String tn = (String) MethodUtils.invokeMethod(record, "getTable"); + TableMetadata tbmd = rdbSchema.findTable(tn); + if (tbmd.isLinkTable()) + continue; + // 节点属性 + Map props = fillProperties(tbmd, record); + Query q = calcNodeWriteQuery(tbmd, props); + queries.add(q); + } + client.reTryRunInTransaction(queries, 5); + + } + + private void doBatchWriteRel(final List writerBuffer) + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + List queries = new ArrayList<>(); + for (Record record : writerBuffer) { + // 表名称 + String tn; + String fk; + + tn = (String) MethodUtils.invokeMethod(record, "getFromTable"); + fk = (String) MethodUtils.invokeMethod(record, "getFk"); + TableMetadata tbmd = rdbSchema.findTable(tn); + // 关系起始表，外键 + TableMetadata from; + TableMetadata to; + String fkName; + Map props; + // 连接表 + if (tbmd.isLinkTable()) { + // 作为起点的外键，连接表有且仅有2外键，其中一个作为起点，另一个为关系终点 + FKConstraintMetadata fromFkmd = tbmd.getLinkFrom(); + from = fromFkmd.getRefTable(); + // + FKConstraintMetadata toFkmd = tbmd.getFks().get(0).getFkName().equals(fromFkmd.getFkName()) + ? tbmd.getFks().get(1) + : tbmd.getFks().get(0); + to = toFkmd.getRefTable(); + fkName = tbmd.getName(); + props = fillProperties(tbmd, record); + Query q = calcLinkRelWriteQuery(from, to, tbmd, fromFkmd, toFkmd, props); + queries.add(q); + } else { + from = tbmd; + FKConstraintMetadata fkmd = from.findFk(fk); + to = fkmd.getRefTable(); + fkName = fkmd.getFkName(); + props = new HashMap(); + Query q = calcRelWriteQuery(from, to, record, fkName, props); + queries.add(q); + } + // 构建查询 + + } + client.reTryRunInTransaction(queries, 5); + } + + // 构建写入 query, 写入模式 + // @WriteMode + private Query calcNodeWriteQuery(TableMetadata tbmd, Map props) { + // insert + String propsStr = ParamsUtils.params2String(props); + String cql = MessageFormat.format(Neo4jQueryPattern.CREATE_NODE, tbmd.getName(), propsStr); + return new Query(cql); + } + + private Query calcRelWriteQuery(TableMetadata from, TableMetadata to, Record record, String fkName, Map props) { + // 连接属性 + String propsStr = ParamsUtils.params2String(props); + // 节点过滤条件，使用主键 + String nodeWherePattern = "{0}.{1} = {2}"; + List nodeWhereItems = new ArrayList<>(); + // + List pkfs = from.getPk().getFields(); + FKConstraintMetadata fk = from.findFk(fkName); + // + List fkfs = fk.getFkFields(); + // from节点 + int i = 0; + for(FieldMetadata f : pkfs) { + Column col = record.getColumn(i); + String item; + if(col.getType()==Type.INT || col.getType()==Type.LONG || col.getType()==Type.DOUBLE) { + item = MessageFormat.format(nodeWherePattern, "a", f.getName(), col.asLong().toString()); + } else { // 其他，字符增加引号 + item = MessageFormat.format(nodeWherePattern, "a", f.getName(), "'"+col.asString()+"'"); + } + nodeWhereItems.add(item); + i++; + } + // to节点 + for(FKField fkf : fkfs) { + Column col = record.getColumn(i); + String item; + if(col.getType()==Type.INT || col.getType()==Type.LONG || col.getType()==Type.DOUBLE) { + item = MessageFormat.format(nodeWherePattern, "b", fkf.getRefField().getName(), col.asLong().toString()); + } else { // 其他，字符增加引号 + item = MessageFormat.format(nodeWherePattern, "b", fkf.getRefField().getName(), "'"+col.asString()+"'"); + } + nodeWhereItems.add(item); + i++; + + } + String nodeWhere = Utils.relWhere("", nodeWhereItems, " and "); + // + String cql = MessageFormat.format(Neo4jQueryPattern.CREATE_REL, + from.getName(), to.getName(), nodeWhere.toString(), fkName, propsStr); + return new Query(cql); + } + + // 连接表关系 + private Query calcLinkRelWriteQuery(TableMetadata from, TableMetadata to, TableMetadata link, + FKConstraintMetadata fromFkmd, FKConstraintMetadata toFkmd, + Map props) { + // 连接属性 + String propsStr = ParamsUtils.params2String(props); + // 节点过滤条件，使用主键 + String nodeWherePattern = "{0}.{1} = {2}"; + List nodeWhereItems = new ArrayList<>(); + // + List fromFkFs = fromFkmd.getFkFields(); + List toFkFs = toFkmd.getFkFields(); + // from节点 + for(FKField fkf : fromFkFs) { + String item; + // from<-link + // from表对应link外键名称 + String fn = fkf.getRefField().getName(); + String ln = fkf.getField().getName(); + Object v = props.get(ln); + if(v instanceof Long || v instanceof Double) { + item = MessageFormat.format(nodeWherePattern, "a", fn, v.toString()); + } else { // 其他，字符增加引号, + item = MessageFormat.format(nodeWherePattern, "a", fn, "'"+v.toString()+"'"); + } + nodeWhereItems.add(item); + + } + // to节点 + for(FKField fkf : toFkFs) { + String item; + // link->to + // to表对应link外键名称 + String fn = fkf.getRefField().getName(); + String ln = fkf.getField().getName(); + Object v = props.get(ln); + if(v instanceof Long || v instanceof Double) { + item = MessageFormat.format(nodeWherePattern, "b", fn, v.toString()); + } else { // 其他，字符增加引号, + item = MessageFormat.format(nodeWherePattern, "b", fn, "'"+v.toString()+"'"); + } + nodeWhereItems.add(item); + } + String nodeWhere = Utils.relWhere("", nodeWhereItems, " and "); + // from->to + String cql = MessageFormat.format(Neo4jQueryPattern.CREATE_REL, + from.getName(), to.getName(), nodeWhere.toString(), link.getName(), propsStr); + return new Query(cql); + } + + // 节点或关系属性，类型转换 + // 本地缓存，查找 + protected Map fillProperties(TableMetadata tbmd, Record record) { + Map props = new HashMap<>(); + int i = 0; + for(FieldMetadata fmd : tbmd.getFields()) { + String columnName = fmd.getName(); + JDBCType jdbcType = fmd.getType(); + Column column = record.getColumn(i); + /* BAD, NULL, INT, LONG, DOUBLE, STRING, BOOL, DATE, BYTES */ + switch (column.getType()) { + case INT: + case LONG: + props.put(columnName, column.asLong()); + break; + case DOUBLE: + props.put(columnName, column.asDouble()); + break; + case STRING: + // 转义 + String col = Utils.strFieldEscape(column.asString()); + props.put(columnName, col); + break; + case BOOL: + props.put(fmd.getName(), column.asBoolean()); + break; + case DATE: + Date date = column.asDate(); + // LocalDateTime ldt = Utils.dateToLocalDateTime(date); + props.put(fmd.getName(), date); + break; + case BYTES: + log.warn(String.format("neo4j不支持二进制属性类型, 字段名:[%s], 字段类型:[%s]. ", + fmd.getName(), + jdbcType.getName())); + break; + default: // 其他，不支持类型 + throw DataXException.asDataXException(Neo4jWriterErrorCode.UNSUPPORTED_TYPE, + String.format("neo4j不支持属性类型, 字段名:[%s], 字段类型:[%s]. ", + fmd.getName(), + jdbcType.getName())); + } // end switch + i++; // 下一个字段 + } // end for tbmd + return props; + } + + // 后处理 + @Override + public void post() { + } + + // 释放 + @Override + public void destroy() { + + } + } +} diff --git a/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Neo4jWriterErrorCode.java b/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Neo4jWriterErrorCode.java new file mode 100644 index 00000000..87cb95a9 --- /dev/null +++ b/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Neo4jWriterErrorCode.java @@ -0,0 +1,36 @@ +package com.leehom.arch.datax.plugin.rdb2graph.writer.neo4jwriter; + +import com.alibaba.datax.common.spi.ErrorCode; + +public enum Neo4jWriterErrorCode implements ErrorCode { + BAD_CONFIG_VALUE("Neo4jWriter-00", "配置的值不合法."), + ERROR_LOAD_RDBSCHEMA("Neo4jWriter-01", "载入关系模式异常."), + UNSUPPORTED_TYPE("Neo4jWriter-02", "不支持字段类型."), + WRONG_RECORD_TYPE("Neo4jWriter-03, {}, {}", "错误记录类型."), + WRONG_NEO4j_CLIENT("Neo4jWriter-04", "neo4j client访问异常."), + ; + + private final String code; + private final String description; + + Neo4jWriterErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s]. ", this.code, + this.description); + } +} \ No newline at end of file diff --git a/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Utils.java b/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Utils.java new file mode 100644 index 00000000..cfb59bf0 --- /dev/null +++ b/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/Utils.java @@ -0,0 +1,61 @@ +package com.leehom.arch.datax.plugin.rdb2graph.writer.neo4jwriter; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.leehom.arch.datax.plugin.rdb2graph.common.StringUtils; + +public class Utils { + + /** 字段分割*/ + public static final String FIELD_SEQ = " and "; + + // 关系wehre + public static String relWhere(List fields) { + return relWhere(null, fields); + } + + public static String relWhere(String prefix, List fields) { + return relWhere(prefix, fields, FIELD_SEQ); + } + + public static String relWhere(String prefix, List fields, String seq) { + if (fields == null || fields.size() == 0) + return ""; + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < fields.size(); i++) { + String tmp = StringUtils.isNotEmpty(prefix) ? prefix+"."+fields.get(i) : fields.get(i) + seq; + sb.append(tmp); + } + // 去掉最后seq + sb.delete(sb.length()-seq.length(), sb.length()); + return sb.toString(); + } + + private static Map ESCAPES = new HashMap(); + + static { + // '\' -> '\\' + // '"' -> '""' + ESCAPES.put("\\", "\\\\\\\\"); + ESCAPES.put("\"", "\"\""); + } + + // 字符字段转义 + public static String strFieldEscape(String fieldString) { + if (StringUtils.isEmpty(fieldString)) + return fieldString; + String ed = fieldString; + for (String key : ESCAPES.keySet()) { + if(ed.contains(key)) { + if("\\".equals(key)) + ed = ed.replaceAll("\\\\", ESCAPES.get(key)); + else + ed = ed.replaceAll(key, ESCAPES.get(key)); + } + } + return ed; + } + +} diff --git a/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/WriteMode.java b/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/WriteMode.java new file mode 100644 index 00000000..c8e00b4a --- /dev/null +++ b/rdb2graph/neo4jwriter/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/writer/neo4jwriter/WriteMode.java @@ -0,0 +1,26 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.writer.neo4jwriter; + +/** + * @类名: WriteMode + * @说明: 写入模式 + * + * @author leehom + * @Date 2020年9月3日 下午6:01:12s + * 修改记录： + * + * @see + */ +public enum WriteMode { + + // CLEAR_BEFORE_INSERT, // 清除 + INSERT, // 直接写入 + INSERT_NOT_EXIST, + REPALCE, // 每次删除后插入 + UPDATE // 更新模式, 不存在插入 + ; + +} diff --git a/rdb2graph/neo4jwriter/src/main/resources/plugin.json b/rdb2graph/neo4jwriter/src/main/resources/plugin.json new file mode 100644 index 00000000..2b4bff47 --- /dev/null +++ b/rdb2graph/neo4jwriter/src/main/resources/plugin.json @@ -0,0 +1,6 @@ +{ + "name": "neo4jwriter", + "class": "com.leehom.arch.datax.plugin.rdb2graph.writer.neo4jwriter.Neo4jWriter", + "description": "", + "developer": "leehom" +} \ No newline at end of file diff --git a/rdb2graph/neo4jwriter/src/test/java/com/leehom/arch/datax/rdb2graph/writer/neo4jwriter/EscapeTest.java b/rdb2graph/neo4jwriter/src/test/java/com/leehom/arch/datax/rdb2graph/writer/neo4jwriter/EscapeTest.java new file mode 100644 index 00000000..564a885e --- /dev/null +++ b/rdb2graph/neo4jwriter/src/test/java/com/leehom/arch/datax/rdb2graph/writer/neo4jwriter/EscapeTest.java @@ -0,0 +1,13 @@ +package com.leehom.arch.datax.rdb2graph.writer.neo4jwriter; + +import com.leehom.arch.datax.plugin.rdb2graph.writer.neo4jwriter.Utils; + +public class EscapeTest { + + public static void main(String[] args) { + String escapeStr = "xx\\Gxx"; + String ed =Utils.strFieldEscape(escapeStr); + System.out.print(ed); + + } +} diff --git a/rdb2graph/oraclereader4graph/oraclereader.md b/rdb2graph/oraclereader4graph/oraclereader.md new file mode 100644 index 00000000..bf35ff72 --- /dev/null +++ b/rdb2graph/oraclereader4graph/oraclereader.md @@ -0,0 +1,350 @@ + +# OracleReader 插件文档 + + +___ + + +## 1 快速介绍 + +OracleReader插件实现了从Oracle读取数据。在底层实现上，OracleReader通过JDBC连接远程Oracle数据库，并执行相应的sql语句将数据从Oracle库中SELECT出来。 + +## 2 实现原理 + +简而言之，OracleReader通过JDBC连接器连接到远程的Oracle数据库，并根据用户配置的信息生成查询SELECT SQL语句并发送到远程Oracle数据库，并将该SQL执行返回结果使用DataX自定义的数据类型拼装为抽象的数据集，并传递给下游Writer处理。 + +对于用户配置Table、Column、Where的信息，OracleReader将其拼接为SQL语句发送到Oracle数据库；对于用户配置querySql信息，Oracle直接将其发送到Oracle数据库。 + + +## 3 功能说明 + +### 3.1 配置样例 + +* 配置一个从Oracle数据库同步抽取数据到本地的作业: + +``` +{ + "job": { + "setting": { + "speed": { + //设置传输速度 byte/s 尽量逼近这个速度但是不高于它. + // channel 表示通道数量，byte表示通道速度，如果单通道速度1MB，配置byte为1048576表示一个channel + "byte": 1048576 + }, + //出错限制 + "errorLimit": { + //先选择record + "record": 0, + //百分比 1表示100% + "percentage": 0.02 + } + }, + "content": [ + { + "reader": { + "name": "oraclereader", + "parameter": { + // 数据库连接用户名 + "username": "root", + // 数据库连接密码 + "password": "root", + "column": [ + "id","name" + ], + //切分主键 + "splitPk": "db_id", + "connection": [ + { + "table": [ + "table" + ], + "jdbcUrl": [ + "jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]" + ] + } + ] + } + }, + "writer": { + //writer类型 + "name": "streamwriter", + // 是否打印内容 + "parameter": { + "print": true + } + } + } + ] + } +} + +``` + +* 配置一个自定义SQL的数据库同步任务到本地内容的作业： + +``` +{ + "job": { + "setting": { + "speed": { + "channel": 5 + } + }, + "content": [ + { + "reader": { + "name": "oraclereader", + "parameter": { + "username": "root", + "password": "root", + "where": "", + "connection": [ + { + "querySql": [ + "select db_id,on_line_flag from db_info where db_id < 10" + ], + "jdbcUrl": [ + "jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]" + ] + } + ] + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "visible": false, + "encoding": "UTF-8" + } + } + } + ] + } +} +``` + + +### 3.2 参数说明 + +* **jdbcUrl** + + * 描述：描述的是到对端数据库的JDBC连接信息，使用JSON的数组描述，并支持一个库填写多个连接地址。之所以使用JSON数组描述连接信息，是因为阿里集团内部支持多个IP探测，如果配置了多个，OracleReader可以依次探测ip的可连接性，直到选择一个合法的IP。如果全部连接失败，OracleReader报错。 注意，jdbcUrl必须包含在connection配置单元中。对于阿里集团外部使用情况，JSON数组填写一个JDBC连接即可。 + + jdbcUrl按照Oracle官方规范，并可以填写连接附件控制信息。具体请参看[Oracle官方文档](http://www.oracle.com/technetwork/database/enterprise-edition/documentation/index.html)。 + + * 必选：是
+ + * 默认值：无
+ +* **username** + + * 描述：数据源的用户名
+ + * 必选：是
+ + * 默认值：无
+ +* **password** + + * 描述：数据源指定用户名的密码
+ + * 必选：是
+ + * 默认值：无
+ +* **table** + + * 描述：所选取的需要同步的表。使用JSON的数组描述，因此支持多张表同时抽取。当配置为多张表时，用户自己需保证多张表是同一schema结构，OracleReader不予检查表是否同一逻辑表。注意，table必须包含在connection配置单元中。
+ + * 必选：是
+ + * 默认值：无
+ +* **column** + + * 描述：所配置的表中需要同步的列名集合，使用JSON的数组描述字段信息。用户使用\*代表默认使用所有列配置，例如['\*']。 + + 支持列裁剪，即列可以挑选部分列进行导出。 + + 支持列换序，即列可以不按照表schema信息进行导出。 + + 支持常量配置，用户需要按照JSON格式: + ["id", "`table`", "1", "'bazhen.csy'", "null", "to_char(a + 1)", "2.3" , "true"] + id为普通列名，\`table\`为包含保留在的列名，1为整形数字常量，'bazhen.csy'为字符串常量，null为空指针，to_char(a + 1)为表达式，2.3为浮点数，true为布尔值。 + + Column必须显示填写，不允许为空！ + + * 必选：是
+ + * 默认值：无
+ +* **splitPk** + + * 描述：OracleReader进行数据抽取时，如果指定splitPk，表示用户希望使用splitPk代表的字段进行数据分片，DataX因此会启动并发任务进行数据同步，这样可以大大提供数据同步的效能。 + + 推荐splitPk用户使用表主键，因为表主键通常情况下比较均匀，因此切分出来的分片也不容易出现数据热点。 + + 目前splitPk仅支持整形、字符串型数据切分，`不支持浮点、日期等其他类型`。如果用户指定其他非支持类型，OracleReader将报错！ + + splitPk如果不填写，将视作用户不对单表进行切分，OracleReader使用单通道同步全量数据。 + + * 必选：否
+ + * 默认值：无
+ +* **where** + + * 描述：筛选条件，MysqlReader根据指定的column、table、where条件拼接SQL，并根据这个SQL进行数据抽取。在实际业务场景中，往往会选择当天的数据进行同步，可以将where条件指定为gmt_create > $bizdate 。注意：不可以将where条件指定为limit 10，limit不是SQL的合法where子句。
+ + where条件可以有效地进行业务增量同步。 + + * 必选：否
+ + * 默认值：无
+ +* **querySql** + + * 描述：在有些业务场景下，where这一配置项不足以描述所筛选的条件，用户可以通过该配置型来自定义筛选SQL。当用户配置了这一项之后，DataX系统就会忽略table，column这些配置型，直接使用这个配置项的内容对数据进行筛选，例如需要进行多表join后同步数据，使用select a,b from table_a join table_b on table_a.id = table_b.id
+ + `当用户配置querySql时，OracleReader直接忽略table、column、where条件的配置`。 + + * 必选：否
+ + * 默认值：无
+ +* **fetchSize** + + * 描述：该配置项定义了插件和数据库服务器端每次批量数据获取条数，该值决定了DataX和服务器端的网络交互次数，能够较大的提升数据抽取性能。
+ + `注意，该值过大(>2048)可能造成DataX进程OOM。`。 + + * 必选：否
+ + * 默认值：1024
+ +* **session** + + * 描述：控制写入数据的时间格式，时区等的配置，如果表中有时间字段，配置该值以明确告知写入 oracle 的时间格式。通常配置的参数为：NLS_DATE_FORMAT,NLS_TIME_FORMAT。其配置的值为 json 格式，例如： +``` +"session": [ + "alter session set NLS_DATE_FORMAT='yyyy-mm-dd hh24:mi:ss'", + "alter session set NLS_TIMESTAMP_FORMAT='yyyy-mm-dd hh24:mi:ss'", + "alter session set NLS_TIMESTAMP_TZ_FORMAT='yyyy-mm-dd hh24:mi:ss'", + "alter session set TIME_ZONE='US/Pacific'" + ] +``` + `(注意"是 " 的转义字符串)`。 + + * 必选：否
+ + * 默认值：无
+ + +### 3.3 类型转换 + +目前OracleReader支持大部分Oracle类型，但也存在部分个别类型没有支持的情况，请注意检查你的类型。 + +下面列出OracleReader针对Oracle类型转换列表: + + +| DataX 内部类型| Oracle 数据类型 | +| -------- | ----- | +| Long |NUMBER,INTEGER,INT,SMALLINT| +| Double |NUMERIC,DECIMAL,FLOAT,DOUBLE PRECISION,REAL| +| String |LONG,CHAR,NCHAR,VARCHAR,VARCHAR2,NVARCHAR2,CLOB,NCLOB,CHARACTER,CHARACTER VARYING,CHAR VARYING,NATIONAL CHARACTER,NATIONAL CHAR,NATIONAL CHARACTER VARYING,NATIONAL CHAR VARYING,NCHAR VARYING | +| Date |TIMESTAMP,DATE | +| Boolean |bit, bool | +| Bytes |BLOB,BFILE,RAW,LONG RAW | + + + +请注意: + +* `除上述罗列字段类型外，其他类型均不支持`。 + + +## 4 性能报告 + +### 4.1 环境准备 + +#### 4.1.1 数据特征 + +为了模拟线上真实数据，我们设计两个Oracle数据表，分别为: + +#### 4.1.2 机器参数 + +* 执行DataX的机器参数为: + +* Oracle数据库机器参数为: + +### 4.2 测试报告 + +#### 4.2.1 表1测试报告 + + +| 并发任务数| DataX速度(Rec/s)|DataX流量|网卡流量|DataX运行负载|DB运行负载| +|--------| --------|--------|--------|--------|--------| +|1| DataX 统计速度(Rec/s)|DataX统计流量|网卡流量|DataX运行负载|DB运行负载| + +## 5 约束限制 + +### 5.1 主备同步数据恢复问题 + +主备同步问题指Oracle使用主从灾备，备库从主库不间断通过binlog恢复数据。由于主备数据同步存在一定的时间差，特别在于某些特定情况，例如网络延迟等问题，导致备库同步恢复的数据与主库有较大差别，导致从备库同步的数据不是一份当前时间的完整镜像。 + +针对这个问题，我们提供了preSql功能，该功能待补充。 + +### 5.2 一致性约束 + +Oracle在数据存储划分中属于RDBMS系统，对外可以提供强一致性数据查询接口。例如当一次同步任务启动运行过程中，当该库存在其他数据写入方写入数据时，OracleReader完全不会获取到写入更新数据，这是由于数据库本身的快照特性决定的。关于数据库快照特性，请参看[MVCC Wikipedia](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) + +上述是在OracleReader单线程模型下数据同步一致性的特性，由于OracleReader可以根据用户配置信息使用了并发数据抽取，因此不能严格保证数据一致性：当OracleReader根据splitPk进行数据切分后，会先后启动多个并发任务完成数据同步。由于多个并发任务相互之间不属于同一个读事务，同时多个并发任务存在时间间隔。因此这份数据并不是`完整的`、`一致的`数据快照信息。 + +针对多线程的一致性快照需求，在技术上目前无法实现，只能从工程角度解决，工程化的方式存在取舍，我们提供几个解决思路给用户，用户可以自行选择： + +1. 使用单线程同步，即不再进行数据切片。缺点是速度比较慢，但是能够很好保证一致性。 + +2. 关闭其他数据写入方，保证当前数据为静态数据，例如，锁表、关闭备库同步等等。缺点是可能影响在线业务。 + +### 5.3 数据库编码问题 + + +OracleReader底层使用JDBC进行数据抽取，JDBC天然适配各类编码，并在底层进行了编码转换。因此OracleReader不需用户指定编码，可以自动获取编码并转码。 + +对于Oracle底层写入编码和其设定的编码不一致的混乱情况，OracleReader对此无法识别，对此也无法提供解决方案，对于这类情况，`导出有可能为乱码`。 + +### 5.4 增量数据同步 + +OracleReader使用JDBC SELECT语句完成数据抽取工作，因此可以使用SELECT...WHERE...进行增量数据抽取，方式有多种： + +* 数据库在线应用写入数据库时，填充modify字段为更改时间戳，包括新增、更新、删除(逻辑删)。对于这类应用，OracleReader只需要WHERE条件跟上一同步阶段时间戳即可。 +* 对于新增流水型数据，OracleReader可以WHERE条件后跟上一阶段最大自增ID即可。 + +对于业务上无字段区分新增、修改数据情况，OracleReader也无法进行增量数据同步，只能同步全量数据。 + +### 5.5 Sql安全性 + +OracleReader提供querySql语句交给用户自己实现SELECT抽取语句，OracleReader本身对querySql不做任何安全性校验。这块交由DataX用户方自己保证。 + +## 6 FAQ + +*** + +**Q: OracleReader同步报错，报错信息为XXX** + + A: 网络或者权限问题，请使用Oracle命令行测试： + sqlplus username/password@//host:port/sid + + +如果上述命令也报错，那可以证实是环境问题，请联系你的DBA。 + + +**Q: OracleReader抽取速度很慢怎么办？** + + A: 影响抽取时间的原因大概有如下几个：(来自专业 DBA 卫绾) + 1. 由于SQL的plan异常，导致的抽取时间长； 在抽取时，尽可能使用全表扫描代替索引扫描; + 2. 合理sql的并发度，减少抽取时间；根据表的大小， + <50G可以不用并发， + <100G添加如下hint: parallel(a,2）, + >100G添加如下hint : parallel(a,4); + 3. 抽取sql要简单，尽量不用replace等函数，这个非常消耗cpu，会严重影响抽取速度; diff --git a/rdb2graph/oraclereader4graph/pom.xml b/rdb2graph/oraclereader4graph/pom.xml new file mode 100644 index 00000000..f044e904 --- /dev/null +++ b/rdb2graph/oraclereader4graph/pom.xml @@ -0,0 +1,90 @@ + + + + com.leehom.arch.datax.plugin + rdb2graph-parent + ${revision} + + 4.0.0 + + oraclereader4graph + jar + + + + com.alibaba.datax + datax-common + ${datax.version} + + + slf4j-log4j12 + org.slf4j + + + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + + + com.oracle + ojdbc6 + 11.2.0.3 + system + ${basedir}/src/main/lib/ojdbc6-11.2.0.3.jar + + + + com.leehom.arch.datax.plugin + rdb2graph-scanner + 1.0.0-SNAPSHOT + + + com.leehom.arch.datax.plugin + rdb2graph-rdbms-util + 1.0.0-SNAPSHOT + + + + + + + + maven-compiler-plugin + + ${java.version} + ${java.version} + ${project-sourceEncoding} + + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + + diff --git a/rdb2graph/oraclereader4graph/src/main/assembly/package.xml b/rdb2graph/oraclereader4graph/src/main/assembly/package.xml new file mode 100644 index 00000000..8c681e5c --- /dev/null +++ b/rdb2graph/oraclereader4graph/src/main/assembly/package.xml @@ -0,0 +1,42 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/reader/oraclereader4graph + + + src/main/lib + + ojdbc6-11.2.0.3.jar + + plugin/reader/oraclereader4graph/libs + + + target/ + + oraclereader4graph-1.0.0-SNAPSHOT.jar + + plugin/reader/oraclereader4graph + + + + + + false + plugin/reader/oraclereader4graph/libs + runtime + + + diff --git a/rdb2graph/oraclereader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/oraclereader/Constant.java b/rdb2graph/oraclereader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/oraclereader/Constant.java new file mode 100644 index 00000000..099f1795 --- /dev/null +++ b/rdb2graph/oraclereader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/oraclereader/Constant.java @@ -0,0 +1,7 @@ +package com.leehom.arch.datax.plugin.rdb2graph.reader.oraclereader; + +public class Constant { + + public static final int DEFAULT_FETCH_SIZE = 1024; + +} diff --git a/rdb2graph/oraclereader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/oraclereader/OracleReader4Graph.java b/rdb2graph/oraclereader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/oraclereader/OracleReader4Graph.java new file mode 100644 index 00000000..4c95a666 --- /dev/null +++ b/rdb2graph/oraclereader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/oraclereader/OracleReader4Graph.java @@ -0,0 +1,143 @@ +package com.leehom.arch.datax.plugin.rdb2graph.reader.oraclereader; + +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.common.spi.Reader; +import com.alibaba.datax.common.util.Configuration; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.CommonRdbms2GraphReader; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Key; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.util.HintUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtilErrorCode; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DataBaseType; + +/** + * @类名: OracleReader4Graph + * @说明: oracle读入 + * 1.表数据同步，datax table模式，schema表写入datax的配置 + 2.表关系同步，按表连接图分组，一对关系一组，表连接关系生成querySql，写入datax的querySql模式配置 + + * + * @author leehom + * @Date 2022年4月28日 下午6:30:15 + * 修改记录： + * + * @see + */ +public class OracleReader4Graph extends Reader { + + private static final DataBaseType DATABASE_TYPE = DataBaseType.Oracle; + + public static class Job extends Reader.Job { + private static final Logger LOG = LoggerFactory.getLogger(OracleReader4Graph.Job.class); + + // + private Configuration originalConfig = null; + private CommonRdbms2GraphReader.Job commonRdbmsReaderJob; + + @Override + public void init() { + // 载入配置 + this.originalConfig = super.getPluginJobConf(); + // + dealFetchSize(this.originalConfig); + // + this.commonRdbmsReaderJob = new CommonRdbms2GraphReader.Job(DATABASE_TYPE); + // + this.commonRdbmsReaderJob.init(this.originalConfig); + + // 注意：要在 this.commonRdbmsReaderJob.init(this.originalConfig); 之后执行，这样可以直接快速判断是否是querySql 模式 + dealHint(this.originalConfig); + } + + // 检查连接/表是否可查询 + @Override + public void preCheck(){ + init(); + // 检测 + this.commonRdbmsReaderJob.preCheck(this.originalConfig, DATABASE_TYPE); + } + + // 分片，包括多表分配，表分片 + @Override + public List split(int adviceNumber) { + return this.commonRdbmsReaderJob.split(this.originalConfig, adviceNumber); + } + + @Override + public void post() { + this.commonRdbmsReaderJob.post(this.originalConfig); + } + + @Override + public void destroy() { + this.commonRdbmsReaderJob.destroy(this.originalConfig); + } + + // fetch size + private void dealFetchSize(Configuration originalConfig) { + int fetchSize = originalConfig.getInt( + com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Constant.FETCH_SIZE, + Constant.DEFAULT_FETCH_SIZE); + if (fetchSize < 1) { + throw DataXException + .asDataXException(DBUtilErrorCode.REQUIRED_VALUE, + String.format("您配置的 fetchSize 有误，fetchSize:[%d] 值不能小于 1.", + fetchSize)); + } + originalConfig.set( + com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Constant.FETCH_SIZE, fetchSize); + } + + private void dealHint(Configuration originalConfig) { + String hint = originalConfig.getString(Key.HINT); + if (StringUtils.isNotBlank(hint)) { + boolean isTableMode = originalConfig.getBool(com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Constant.IS_TABLE_MODE).booleanValue(); + if(!isTableMode){ + throw DataXException.asDataXException(OracleReaderErrorCode.HINT_ERROR, "当且仅当非 querySql 模式读取 oracle 时才能配置 HINT."); + } + HintUtil.initHintConf(DATABASE_TYPE, originalConfig); + } + } + } + + public static class Task extends Reader.Task { + + private Configuration readerSliceConfig; + private CommonRdbms2GraphReader.Task commonRdbmsReaderTask; + + @Override + public void init() { + this.readerSliceConfig = super.getPluginJobConf(); + this.commonRdbmsReaderTask = new CommonRdbms2GraphReader.Task( + DATABASE_TYPE ,super.getTaskGroupId(), super.getTaskId()); + this.commonRdbmsReaderTask.init(this.readerSliceConfig); + } + + @Override + public void startRead(RecordSender recordSender) { + int fetchSize = this.readerSliceConfig + .getInt(com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Constant.FETCH_SIZE); + + this.commonRdbmsReaderTask.startRead(this.readerSliceConfig, + recordSender, super.getTaskPluginCollector(), fetchSize); + } + + @Override + public void post() { + this.commonRdbmsReaderTask.post(this.readerSliceConfig); + } + + @Override + public void destroy() { + this.commonRdbmsReaderTask.destroy(this.readerSliceConfig); + } + + } + +} diff --git a/rdb2graph/oraclereader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/oraclereader/OracleReaderErrorCode.java b/rdb2graph/oraclereader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/oraclereader/OracleReaderErrorCode.java new file mode 100644 index 00000000..f4441011 --- /dev/null +++ b/rdb2graph/oraclereader4graph/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/reader/oraclereader/OracleReaderErrorCode.java @@ -0,0 +1,32 @@ +package com.leehom.arch.datax.plugin.rdb2graph.reader.oraclereader; + +import com.alibaba.datax.common.spi.ErrorCode; + +public enum OracleReaderErrorCode implements ErrorCode { + HINT_ERROR("Oraclereader-00", "您的 Hint 配置出错."), + ; + + private final String code; + private final String description; + + private OracleReaderErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s]. ", this.code, + this.description); + } +} diff --git a/rdb2graph/oraclereader4graph/src/main/resources/plugin.json b/rdb2graph/oraclereader4graph/src/main/resources/plugin.json new file mode 100644 index 00000000..eab3d3bf --- /dev/null +++ b/rdb2graph/oraclereader4graph/src/main/resources/plugin.json @@ -0,0 +1,6 @@ +{ + "name": "oraclereader4graph", + "class": "com.leehom.arch.datax.plugin.rdb2graph.reader.oraclereader.OracleReader4Graph", + "description": "", + "developer": "leehom" +} \ No newline at end of file diff --git a/rdb2graph/oraclereader4graph/src/main/resources/plugin_job_template.json b/rdb2graph/oraclereader4graph/src/main/resources/plugin_job_template.json new file mode 100644 index 00000000..cc06128a --- /dev/null +++ b/rdb2graph/oraclereader4graph/src/main/resources/plugin_job_template.json @@ -0,0 +1,16 @@ +{ + "name": "oraclereader4graph", + "parameter": { + "username": "", + "password": "", + "phase": "", + "schemaUri": "", + "column": [], + "connection": [ + { + "table": [], + "jdbcUrl": [] + } + ] + } +} \ No newline at end of file diff --git a/rdb2graph/pom.xml b/rdb2graph/pom.xml new file mode 100644 index 00000000..e222d0f4 --- /dev/null +++ b/rdb2graph/pom.xml @@ -0,0 +1,146 @@ + + + 4.0.0 + + com.alibaba.datax + datax-all + 0.0.1-SNAPSHOT + + + com.leehom.arch.datax.plugin + rdb2graph-parent + pom + ${revision} + + + 1.0.0-SNAPSHOT + 1.8 + 1.2.36 + 1.4.10 + 27.0.1-jre + 0.0.1-SNAPSHOT + 1.18.12 + 1.1.23 + 1.18.12 + 1.0.1-SNAPSHOT + + + + + rdb2graph-scanner + rdb2graph-transformer + neo4jwriter + mysqlreader4graph + oraclereader4graph + rdb2grpah-rdbms-util + rdb2graph-common + rdb2graph-datax + + + + + rdc-releases + https://repo.rdc.aliyun.com/repository/128947-release-JP7RVf/ + + + rdc-snapshots + https://repo.rdc.aliyun.com/repository/128947-snapshot-2gBBfM/ + + + + + + + com.alibaba + fastjson + ${fastjson.version} + + + com.alibaba.datax + datax-core + 0.0.1-SNAPSHOT + + + com.alibaba.datax + datax-common + 0.0.1-SNAPSHOT + + + org.projectlombok + lombok + ${lombok.version} + provided + + + com.alibaba + druid + ${druid.version} + + + com.google.guava + guava + ${guava.version} + + + com.thoughtworks.xstream + xstream + ${xstream.version} + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${java.version} + ${java.version} + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + true + + + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.1.0 + + + true + oss + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + + + \ No newline at end of file diff --git a/rdb2graph/rdb2graph-common/pom.xml b/rdb2graph/rdb2graph-common/pom.xml new file mode 100644 index 00000000..a7e9b567 --- /dev/null +++ b/rdb2graph/rdb2graph-common/pom.xml @@ -0,0 +1,54 @@ + + 4.0.0 + + com.leehom.arch.datax.plugin + rdb2graph-parent + ${revision} + + rdb2graph-common + jar + + + 0.9.0 + UTF-8 + + + + + + com.google.guava + guava + + + + org.projectlombok + lombok + + + + com.alibaba.datax + datax-core + + + com.thoughtworks.xstream + xstream + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + logback.xml + application.yml + + + + + + diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/BeanUtils.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/BeanUtils.java new file mode 100644 index 00000000..6da1aebf --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/BeanUtils.java @@ -0,0 +1,158 @@ +/** + * + */ +package com.leehom.arch.datax.plugin.rdb2graph.common; + +import java.io.PrintStream; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Date; +import java.util.Map; + +/** + * 类名: BeanUtils + * 说明: Bean工具类 + * + * @author leehom + * @Date Nov 30, 2009 4:09:36 PM + * 修改记录： + * + * @see + */ +public class BeanUtils { + + /** + * @说明：输出Bean的属性/深输出Bean的属性 + * + * @param '输出流' + * @author hjli + * + * @异常： + */ + public static void printBean(Object bean) { + printBean(bean, System.out); + } + + public static void printBean(Object bean, PrintStream out) { + out.println(bean.toString()); + } + + public static void printBeanDeep(Object bean) { + printBeanDeep(bean, System.out); + } + + public static void printBeanDeep(Object bean, PrintStream out) { + try { + Map m = org.apache.commons.beanutils.BeanUtils.describe(bean); + for (Object o : m.keySet()) { + if(o==null){ + out.println("Null value field"); + continue; + } + out.println(o.toString()+":"+m.get(o)); + } + } catch(Exception ex) { + throw new RuntimeException(ex.getMessage()); + } + } + + public static Long number2Long(Number num) { + if(num!=null) + return num.longValue(); + return null; + + } + + public static Integer number2Integer(Number num) { + if(num!=null) + return num.intValue(); + return null; + + } + + public static Double number2Double(Number num) { + if(num!=null) + return num.doubleValue(); + return null; + + } + + public static Short number2Short(Number num) { + if(num!=null) + return num.shortValue(); + return null; + + } + + public static Byte number2Byte(Number num) { + if(num!=null) + return num.byteValue(); + return null; + + } + + public static Double bigDecimal2Double(BigDecimal num) { + if(num!=null) + return num.doubleValue(); + return null; + + } + + public static Long bigDecimal2Long(BigDecimal num) { + if(num!=null) + return num.longValue(); + return null; + + } + + public static Integer bigDecimal2Integer(BigDecimal num) { + if(num!=null) + return num.intValue(); + return null; + + } + + public static Integer bigInteger2Integer(BigInteger num) { + if(num!=null) + return num.intValue(); + return null; + + } + + public static Long bigInteger2Long(BigInteger num) { + if(num!=null) + return num.longValue(); + return null; + + } + + public static Date stringToDate(String dateStr, Class clazz) { + if(dateStr==null) + return null; + return DateTimeUtils.StringToDate(dateStr); + + } + + public static double doublePrecision(double d, short precision) { + BigDecimal bg = new BigDecimal(d); + double d2 = bg.setScale(precision, BigDecimal.ROUND_HALF_DOWN).doubleValue(); + return d2; + } + + public static Class getTemplateType(Object obj) { + Class clazz = null; + Class c = obj.getClass(); + Type t = c.getGenericSuperclass(); + if (t instanceof ParameterizedType) { + Type[] p = ((ParameterizedType) t).getActualTypeArguments(); + if(p[0] instanceof ParameterizedType ) + clazz = (Class) ((ParameterizedType) p[0]).getRawType(); + else + clazz = (Class) p[0]; + } + return clazz; + + } +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/ByteAndStreamUtils.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/ByteAndStreamUtils.java new file mode 100644 index 00000000..0ce1775a --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/ByteAndStreamUtils.java @@ -0,0 +1,125 @@ +/** + * %流程框架% + * %1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.common; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; + +/** + * @类名: ByteAndStreamUtils + * @说明: 字节与流工具类 + * + * @author leehom + * @Date 2010-4-2 下午02:19:25 + * 修改记录： + * + * @see + */ +public class ByteAndStreamUtils { + + /** + * @说明：流转换成字节 + * + * @author hjli + * @param is + * @return + * + * @异常： + */ + public static byte[] StreamToBytes(InputStream is) { + BufferedInputStream bis = null; + try { + is = new BufferedInputStream(is); + byte[] bytes = new byte[is.available()]; + int len = bytes.length; + int offset = 0; + int read = 0; + while (offset < len + && (read = is.read(bytes, offset, len - offset)) > 0) { + offset += read; + } + return bytes; + } catch (Exception e) { + return null; + } finally { + try { + is.close(); + is = null; + } catch (IOException e) { + return null; + } + + } + } + // A block of stream to bytes + public static byte[] StreamBlockToBytes(InputStream is, long offset, int size) { + BufferedInputStream bis = null; + try { + is = new BufferedInputStream(is); + // Skip to the position where to start receiving bytes + is.skip(offset); + // Actual data size that would be get + int datSize = is.available()< size ? is.available() : size; + byte[] bytes = new byte[datSize]; + // Offset of data bytes which to start storing bytes + int dataOffset = 0; + int read = 0; + while (dataOffset < size + && (read = is.read(bytes, dataOffset, datSize - dataOffset)) > 0) { + dataOffset += read; + } + return bytes; + } catch (Exception e) { + return null; + } + } + + /** + * 从字节数组获取对象 + * @Author Sean.guo + * @EditTime 2007-8-13 上午11:46:34 + */ + public static Object objectFromBytes(byte[] objBytes) throws IOException, ClassNotFoundException { + if (objBytes == null || objBytes.length == 0) { + return null; + } + ByteArrayInputStream bi = new ByteArrayInputStream(objBytes); + ObjectInputStream oi = new ObjectInputStream(bi); + return oi.readObject(); + } + + /** + * 从对象获取一个字节数组 + * @Author Sean.guo + * @EditTime 2007-8-13 上午11:46:56 + */ + public static byte[] objectToBytes(Serializable obj) throws IOException { + if (obj == null) { + return null; + } + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + ObjectOutputStream oo = new ObjectOutputStream(bo); + oo.writeObject(obj); + return bo.toByteArray(); + } + + // 深克隆对象 + public static Object deepClone(Serializable obj) throws IOException, ClassNotFoundException { + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + ObjectOutputStream oo = new ObjectOutputStream(bo); + oo.writeObject(obj); + // + ByteArrayInputStream bi = new ByteArrayInputStream(bo.toByteArray()); + ObjectInputStream oi = new ObjectInputStream(bi); + return oi.readObject(); + } + +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/DateTimeUtils.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/DateTimeUtils.java new file mode 100644 index 00000000..ea6662f3 --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/DateTimeUtils.java @@ -0,0 +1,306 @@ +package com.leehom.arch.datax.plugin.rdb2graph.common; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 类名: DateTimeUtils.java + * 说明: 日期时间工具类 + * + * @author leehom + * @Date Apr 23, 2009 5:36:41 AM + * + * @see + */ +public class DateTimeUtils { + + /** + * 默认数据模式 + */ + public static final String defaultPatten = "yyyy-MM-dd HH:mm:ss"; + public static final String defaultPattenMillis = "yyyy-MM-dd HH:mm:ss.SSS"; + public static final String defaultDatePatten = "yyyyMMdd"; + public static final String DatePatten2 = "yyyy-MM-dd"; + // ISO8601 Date Type Pattern + public static final String ISO8601Patten = "yyyy-MM-dd'T'HH:mm:ssZZ"; + public static final String ISO8601PattenNoZone = "yyyy-MM-dd'T'HH:mm:ss"; + public static final String ISO8601PattenWithMillis = "yyyy-MM-dd'T'HH:mm:ss.SSSZZ"; + + + /** + * 说明：Date to String + * + * @author hjli + * @Param @param date + * @Param @return + * @Return String + * + */ + public static String DateToString(Date date) { + return DateToString(date, defaultPatten); + } + + public static String DateToString(Date date, String pattern) { + SimpleDateFormat sdf = new SimpleDateFormat(pattern); + return sdf.format(date); + } + + /** + * @说明：String to Date + * + * @author leehong + * @param dateStr + * @return Date + * + * @异常： + */ + public static Date StringToDate(String dateStr) { + return StringToDate(dateStr, defaultPatten); + } + + public static Date StringToDate(String dateStr, String pattern) { + if(dateStr==null) + return null; + SimpleDateFormat sdf = new SimpleDateFormat(pattern); + try { + return sdf.parse(dateStr); + } catch (Exception ex) { + ex.printStackTrace(); + return null; + } + } + + /** + * 获取当前日期 + * + * @author hjli + * @Param @return + * @Return String + * + * TODO + * + */ + public static String getCurrentDateTimeText(String pattern) { + if ((pattern==null)||(pattern.equals(""))) { + pattern = defaultPatten; + } + SimpleDateFormat sdf = new SimpleDateFormat(pattern); + return sdf.format(Calendar.getInstance().getTime()); + } + + public static String getCurrentDateTimeText() { + return getCurrentDateTimeText(defaultPatten); + } + + /** + * 说明：获取前一小时时间 + * + * @author hjli + * @Param @param pattern + * @Param @return + * @Return String + * + * TODO + * + */ + public static String getPreHourText(String pattern) { + Calendar c = Calendar.getInstance(); + return getPreHourText(c.getTime(), pattern); + } + + public static String getPreHourText(Date date, String pattern) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.HOUR_OF_DAY, -1); + SimpleDateFormat sdf = new SimpleDateFormat(pattern); + return sdf.format(c.getTime()); + } + + public static Date getPreHour(Date date) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.HOUR_OF_DAY, -1); + return c.getTime(); + } + + /** 获取前n小时时间*/ + public static Date getPreHourN(Date date, int n) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.HOUR_OF_DAY, 0-n); + return c.getTime(); + } + + public static Date getNextHour(Date date) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.HOUR_OF_DAY, 1); + return c.getTime(); + + } + + public static Date getNextHourN(Date date, int n) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.HOUR_OF_DAY, n); + return c.getTime(); + + } + + /** + * 说明：获取当前日前一日 + * + * @author hjli + * @Param @param pattern + * @Param @return + * @Return String + * + * TODO + * + */ + public static String getPreDayText(String pattern) { + Calendar c = Calendar.getInstance(); + c.add(Calendar.DATE, -1); + SimpleDateFormat sdf = new SimpleDateFormat(pattern); + return sdf.format(c.getTime()); + } + + public static Date getPreDay(Date date) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.DATE, -1); + return c.getTime(); + } + + /** 获取前n天时间*/ + public static Date getPreDayN(Date date, int n) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.DATE, 0-n); + return c.getTime(); + } + + public static Date getNextDay(Date date) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.DATE, 1); + return c.getTime(); + + } + + public static Date getNextDayN(Date date, int n) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.DATE, n); + return c.getTime(); + + } + + public static Date getPreWeek(Date date) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.DATE, -7); + return c.getTime(); + } + + /** + * 说明：获取当前月前一月 + * + * @author hjli + * @Param @param pattern + * @Param @return + * @Return String + * + */ + public static String getPreMonthText(String pattern) { + Calendar c = Calendar.getInstance(); + c.add(Calendar.MONTH, -1); + SimpleDateFormat sdf = new SimpleDateFormat(pattern); + return sdf.format(c.getTime()); + } + + public static Date getPreMonth(Date date) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.MONTH, -1); + return c.getTime(); + } + + /** 获取前n月时间*/ + public static Date getPreMonthN(Date date, int n) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.MONTH, 0-n); + return c.getTime(); + } + + public static Date getNextMonth(Date date) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.MONTH, 1); + return c.getTime(); + + } + + public static Date getNextMonthN(Date date, int n) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.MONTH, n); + return c.getTime(); + + } + + // 获取前一年 + public static String getPreYearText(String pattern) { + Calendar c = Calendar.getInstance(); + c.add(Calendar.YEAR, -1); + SimpleDateFormat sdf = new SimpleDateFormat(pattern); + return sdf.format(c.getTime()); + } + + public static Date getPreYear(Date date) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.YEAR, -1); + return c.getTime(); + } + + /** 获取前n年*/ + public static Date getPreYearN(Date date, int n) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.YEAR, 0-n); + return c.getTime(); + } + + /** 获取下一年*/ + public static Date getNextYear(Date date) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.YEAR, 1); + return c.getTime(); + + } + + public static Date getNextYearN(Date date, int n) { + Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(Calendar.YEAR, n); + return c.getTime(); + + } + + // the right date type should be like "2012-12-15T00:00:00+08:00" + public static void validateDateFormat(String dateString) throws Exception { + Pattern pattern = Pattern.compile("\\d{4}\\-\\d{2}\\-\\d{2}T\\d{2}:\\d{2}:\\d{2}[\\+|\\-]\\d{2}:\\d{2}"); + Matcher matcher = pattern.matcher(dateString); + boolean isValidated = matcher.matches(); + if (!isValidated) { + throw new Exception("'" + dateString + "' is not a validate " + + "date string. Please follow this sample:2012-12-15T00:00:00+08:00"); + } + } +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/RelRecord.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/RelRecord.java new file mode 100644 index 00000000..49ecef30 --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/RelRecord.java @@ -0,0 +1,43 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.common; + +import com.alibaba.datax.core.transport.record.DefaultRecord; + +/** + * @类名: RelRecord + * @说明: 关系record + * + * @author leehom + * @Date 2022年4月26日 下午7:31:18 + * 修改记录： + * + * @see + */ +public class RelRecord extends DefaultRecord { + + /** 关系起始表或者连接表*/ + private String fromTable; + /** + * 若连接表，fk则为连接的起点的关联表 + * 若为一般表，fk则为连接的外键，表可有多个外键 + */ + private String fk; + + public String getFk() { + return fk; + } + public void setFk(String fk) { + this.fk = fk; + } + public String getFromTable() { + return fromTable; + } + public void setFromTable(String fromTable) { + this.fromTable = fromTable; + } + + +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/ResourceLoaderUtil.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/ResourceLoaderUtil.java new file mode 100644 index 00000000..1abddd41 --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/ResourceLoaderUtil.java @@ -0,0 +1,251 @@ +package com.leehom.arch.datax.plugin.rdb2graph.common; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @类名: ResourceLoaderUtil + * @说明: 资源加载工具类 + * + * @author leehom + * @Date 2018-9-9 上午12:05:39 + * 修改记录： + * + * @see + */ +public class ResourceLoaderUtil { + + private static Logger log = LoggerFactory.getLogger(ResourceLoaderUtil.class); + + /** + * + * get a class with the name specified + * + * @paramclassName + * + * @return + * + */ + public static Class loadClass(String className) { + try { + return getClassLoader().loadClass(className); + } + catch (ClassNotFoundException e) { + throw new RuntimeException("class not found '" + className + "'", e); + } + } + + /** + * + * get the class loader + * + * @return + * + */ + public static ClassLoader getClassLoader() { + return ResourceLoaderUtil.class.getClassLoader(); + } + + /** + * + * get the file that from the path that related to current class path + * example, there is the file root like this: + * |--resource/xml/target.xml + * |--classes + * in order to get the xml file the api should called like this: + * getStream("../resource/xml/target.xml"); + * @paramrelativePath + * + * @return + * + * @throwsIOException + * + * @throwsMalformedURLException + * + */ + public static InputStream getResourceStream(String relativePath) throws MalformedURLException, IOException { + if (!relativePath.contains("../")) { + return getClassLoader().getResourceAsStream(relativePath); + } + else { + return ResourceLoaderUtil.getStreamByExtendResource(relativePath); + } + } + + /** + * + * + * + * @paramurl + * + * @return + * + * @throwsIOException + * + */ + public static InputStream getStream(URL url) throws IOException { + if (url != null) { + return url.openStream(); + } + else { + return null; + } + } + + /** + * + * get the file that from the path that related to current class path + * example, there is the file root like this: + * |--resource/xml/target.xml + * |--classes + * in order to get the xml file the api should called like this: + * getStream("../resource/xml/target.xml"); + * @return + * + * @throwsMalformedURLException + * + * @throwsIOException + * + */ + public static InputStream getStreamByExtendResource(String relativePath) throws MalformedURLException, IOException { + return ResourceLoaderUtil.getStream(ResourceLoaderUtil.getExtendResource(relativePath)); + } + + /** + * + * + * @paramresource + * + * @return + * + */ + public static Properties getProperties(String resource) { + Properties properties = new Properties(); + try { + properties.load(getResourceStream(resource)); + } + catch (IOException e) { + throw new RuntimeException("couldn't load properties file '" + resource + "'", e); + } + return properties; + } + + /** + * + * get the absolute path of the class loader of this Class + * + * + * @return + * + */ + public static String getAbsolutePathOfClassLoaderClassPath() { + //ClassLoaderUtil.log.info(ClassLoaderUtil.getClassLoader().getResource("").toString()); + return ResourceLoaderUtil.getClassLoader().getResource("").getPath(); + } + + /** + * + * get the file that from the path that related to current class path + * example, there is the file root like this: + * |--resource/xml/target.xml + * |--classes + * in order to get the xml file the api should called like this: + * getStream("../resource/xml/target.xml"); + * + * @throwsMalformedURLException + * + */ + public static URL getExtendResource(String relativePath) throws MalformedURLException { + //ClassLoaderUtil.log.info("The income relative path:" + relativePath); + // ClassLoaderUtil.log.info(Integer.valueOf(relativePath.indexOf("../"))) ; + if (!relativePath.contains("../")) { + return ResourceLoaderUtil.getResource(relativePath); + } + String classPathAbsolutePath = ResourceLoaderUtil.getAbsolutePathOfClassLoaderClassPath(); + if (relativePath.substring(0, 1).equals("/")) { + relativePath = relativePath.substring(1); + } + //ClassLoaderUtil.log.info(Integer.valueOf(relativePath.lastIndexOf("../"))); + String wildcardString = relativePath.substring(0, relativePath.lastIndexOf("../") + 3); + relativePath = relativePath.substring(relativePath.lastIndexOf("../") + 3); + int containSum = ResourceLoaderUtil.containSum(wildcardString, "../"); + classPathAbsolutePath = ResourceLoaderUtil.cutLastString(classPathAbsolutePath, "/", containSum); + String resourceAbsolutePath = classPathAbsolutePath + relativePath; + //ClassLoaderUtil.log.info("The income absolute path:" + resourceAbsolutePath); + URL resourceAbsoluteURL = new URL(resourceAbsolutePath); + return resourceAbsoluteURL; + } + + /** + * + * + * + * @paramsource + * + * @paramdest + * + * @return + * + */ + private static int containSum(String source, String dest) { + int containSum = 0; + int destLength = dest.length(); + while (source.contains(dest)) { + containSum = containSum + 1; + source = source.substring(destLength); + } + return containSum; + } + + /** + * + * + * + * @paramsource + * + * @paramdest + * + * @paramnum + * + * @return + * + */ + private static String cutLastString(String source, String dest, int num) { + // String cutSource=null; + for (int i = 0; i < num; i++) { + source = source.substring(0, source.lastIndexOf(dest, source.length() - 2) + 1); + } + return source; + } + + /** + * + * + * + * @paramresource + * + * @return + * + */ + public static URL getResource(String resource) { + //ClassLoaderUtil.log.info("The income classpath related path:" + resource); + return ResourceLoaderUtil.getClassLoader().getResource(resource); + } + + public static File getFile(String resource) throws URISyntaxException { + URL url = ResourceLoaderUtil.getClassLoader().getResource(resource); + if(url==null) + return null; + File file=new File(url.getPath()); + return file; + } + +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/StringUtils.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/StringUtils.java new file mode 100644 index 00000000..23cf5a8e --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/StringUtils.java @@ -0,0 +1,221 @@ +/** + * %utils% + * %1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.common; + +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @类名: StringUtils + * @说明: + * + * @author leehom + * @Date 2021年11月27日 下午4:47:54 + * 修改记录： + * + * @see + */ +public class StringUtils { + + public static final String EMPTY = ""; + + /** + * Pads a String s to take up n characters, + * padding with char c on the left (true) or on + * the right (false). Returns null if passed a + * null String. + **/ + public static String paddingString(String s, int n, char c, boolean paddingLeft) { + if (s == null) { + return s; + } + int add = n - s.length(); // may overflow int size... should not be a + // problem in real life + if (add <= 0) { + return s; + } + StringBuffer str = new StringBuffer(s); + char[] ch = new char[add]; + Arrays.fill(ch, c); + if (paddingLeft) { + str.insert(0, ch); + } else { + str.append(ch); + } + return str.toString(); + } + + public static String padLeft(String s, int n, char c) { + return paddingString(s, n, c, true); + } + + public static String padRight(String s, int n, char c) { + return paddingString(s, n, c, false); + } + + public static boolean isInt(String s) { + try { + Integer.parseInt(s); + return true; + } catch (Exception ex) { + return false; + } + } + + /** + * @说明：字符串编码 + * + * @author leehom + * @param src + * @return + * + * 异常： + */ + public static String strCoderGBk(String src) { + try { + if (src == null) + return null; + return new String(src.getBytes("ISO-8859-1"), "utf8"); + } catch (UnsupportedEncodingException e) { + return null; + } + } + + public static String stringFromSet(Set ss, String spliter) { + if (ss == null || ss.size() == 0) + return null; + StringBuffer result = new StringBuffer(); + String[] a = ss.toArray(new String[0]); + // + result.append(a[0]); + for (int i = 1; i < a.length; i++) { + result.append(spliter); + result.append(a[i]); + } + return result.toString(); + } + + // 字符串->字符数组 + public static Set str2Set(String s, String spliter) { + if (s == null || s.equals("")) + return null; + String[] sa = s.split(spliter); + List sl = Arrays.asList(sa); + Set set = new HashSet(sl); + return set; + } + + // 字符串转换字符串数组 + public static String[] str2Strings(String s, String spliter) { + if (s == null || s.equals("")) + return new String[0]; + String[] sa = s.split(spliter); + String[] ss = new String[sa.length]; + for(int i=0;i array2List(String[] strArry) { + return Arrays.asList(strArry); + } + + public static List str2List(String s, String spliter) { + String[] arrayStr = str2Strings(s, spliter); + return Arrays.asList(arrayStr); + } + + public static String array2Str(Object[] objs) { + return array2Str(objs, "\"", ","); + } + + public static String list2Str(List objs, String quotor, String seq) { + if (objs == null || objs.size() == 0) + return ""; + String s = ""; + for (int i = 0; i < objs.size(); i++) { + Object obj = objs.get(i); + if (obj instanceof String) + s = s + quotor + objs.get(i).toString() + quotor; + else + s = s + objs.get(i).toString(); + if (i != objs.size() - 1) + s = s + seq; + } + return s; + } + + public static String array2Str(Object[] objs, String quotor, String seq) { + if (objs == null || objs.length == 0) + return ""; + String s = ""; + for (int i = 0; i < objs.length; i++) { + s = s + quotor + objs[i].toString() + quotor; + if (i != objs.length - 1) + s = s + seq; + } + return s; + } + + // 去除文本中的的html标签 + public static String removeTagFromText(String content) { + Pattern p = null; + Matcher m = null; + String value = null; + // 去掉<>标签 + p = Pattern.compile("(<[^>]*>)"); + m = p.matcher(content); + String temp = content; + while (m.find()) { + value = m.group(0); + temp = temp.replace(value, ""); + } + return temp; + } + + public static String escape(String target, Map escapes) { + Set key = escapes.keySet(); + for (Iterator it = key.iterator(); it.hasNext();) { + String s = (String) it.next(); + target = target.replaceAll(s, escapes.get(s)); + } + return target; + } + + public static boolean isBlank(String str) { + int strLen; + if (str == null || (strLen = str.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if ((!Character.isWhitespace(str.charAt(i)))) { + return false; + } + } + return true; + } + + public static boolean isNotBlank(String str) { + return !isBlank(str); + } + + public static boolean isNotEmpty(String str) { + return str!=null && !"".equals(str); + } + + public static boolean isEmpty(String str) { + return str==null || "".equals(str); + } + +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/TableRecord.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/TableRecord.java new file mode 100644 index 00000000..b4d86a59 --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/TableRecord.java @@ -0,0 +1,33 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.common; + +import com.alibaba.datax.core.transport.record.DefaultRecord; + +/** + * @类名: TableRecord + * @说明: 表record + * + * @author leehom + * @Date 2022年4月26日 下午7:31:18 + * 修改记录： + * + * @see + */ +public class TableRecord extends DefaultRecord { + + /** 记录所属表*/ + private String table; + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/Serializer.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/Serializer.java new file mode 100644 index 00000000..9f0660cc --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/Serializer.java @@ -0,0 +1,48 @@ +/** + * %基础类% + * %1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.common.serializer; + +/** + * @类名: Serializer + * @说明: 序列化器接口 + * + * @author leehom + * @Date 2010-6-9 上午09:58:34 + * 修改记录： + * + * @see + */ + +public interface Serializer { + + public static final String PREFIX_CDATA = ""; + + /** + * @说明：序列化 + * + * @author leehom + * @param obj 目标对象 + * @throws SeriallizeException + * @return void + * + * @异常： + */ + public byte[] Marshal(Object obj) throws SeriallizeException; + + /** + * @说明：反序列化 + * + * @author leehong + * @param xml XML字符串 + * typeAlias 类型关联，即XML与那个Class关联 + * @return + * @throws SeriallizeException + * @return Object + * + * @异常： + */ + public Object Unmarshal(byte[] xml) throws SeriallizeException; +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/SeriallizeException.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/SeriallizeException.java new file mode 100644 index 00000000..235a5cbe --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/SeriallizeException.java @@ -0,0 +1,27 @@ +package com.leehom.arch.datax.plugin.rdb2graph.common.serializer; + + +/** + * @类名: SeriallizeException + * @说明: 序列化基本异常 + * + * @author leehom + * @Date 2010-6-4 下午01:18:20 + * 修改记录： + * + * @see + */ +public class SeriallizeException extends Exception { + + public SeriallizeException() { + super(); + // TODO Auto-generated constructor stub + } + + public SeriallizeException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/xstream/DateConverter.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/xstream/DateConverter.java new file mode 100644 index 00000000..d59c01d7 --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/xstream/DateConverter.java @@ -0,0 +1,86 @@ +/** + * %序列化器% + * %1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.common.serializer.xstream; + +import java.sql.Timestamp; +import java.util.Date; + +import com.leehom.arch.datax.plugin.rdb2graph.common.DateTimeUtils; +import com.thoughtworks.xstream.converters.SingleValueConverter; + +/** + * @类名: DateConverter + * @说明: 日期转换器 + * + * @author leehom + * @Date 2010-6-15 下午06:09:19 + * 修改记录： + * + * @see + */ +public class DateConverter implements SingleValueConverter { + + /** 日期模式*/ + private String pattern; + + /* + * (non-Javadoc) + * + * @see + * com.thoughtworks.xstream.converters.SingleValueConverter#fromString(java + * .lang.String) + */ + + public Object fromString(String str) { + return DateTimeUtils.StringToDate(str, pattern()); + } + + /* + * (non-Javadoc) + * + * @see + * com.thoughtworks.xstream.converters.SingleValueConverter#toString(java + * .lang.Object) + */ + + public String toString(Object obj) { + return DateTimeUtils.DateToString((Date)obj, pattern()); + + } + + private String pattern() { + if(pattern==null) + return DateTimeUtils.defaultPatten; + else + return pattern; + } + + /* + * (non-Javadoc) + * + * @see + * com.thoughtworks.xstream.converters.ConverterMatcher#canConvert(java. + * lang.Class) + */ + + public boolean canConvert(Class type) { + return type.equals(Date.class) || type.equals(Timestamp.class); + } + + /** + * @return the pattern + */ + public String getPattern() { + return pattern; + } + + /** + * @param pattern the pattern to set + */ + public void setPattern(String pattern) { + this.pattern = pattern; + } + +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/xstream/JsonSerializerXStreamImpl.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/xstream/JsonSerializerXStreamImpl.java new file mode 100644 index 00000000..8fe05d48 --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/xstream/JsonSerializerXStreamImpl.java @@ -0,0 +1,172 @@ +/** + * %serializer% + * %1.2% + */ +package com.leehom.arch.datax.plugin.rdb2graph.common.serializer.xstream; + +import java.io.ByteArrayInputStream; +import java.io.UnsupportedEncodingException; +import java.text.MessageFormat; +import java.util.List; +import java.util.Map; + +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.Serializer; +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.SeriallizeException; +import com.thoughtworks.xstream.XStream; +import com.thoughtworks.xstream.converters.Converter; +import com.thoughtworks.xstream.converters.SingleValueConverter; +import com.thoughtworks.xstream.io.json.JettisonMappedXmlDriver; +import com.thoughtworks.xstream.mapper.Mapper; + +/** + * @类名: JsonSerializerXStreamImpl + * @说明: json序列化器XStream实现 + * + * + * @author leehom + * @Date 2010-6-9 上午10:58:40 + * + * + * @see + */ +public class JsonSerializerXStreamImpl implements Serializer { + + /** 序列化库*/ + protected XStream xs; + /** 类型关联*/ + private Map typeAlias; + /** 忽略字段*/ + private Map omitAlias; + /** + * 取消标记, + * @see XStream.addImplicitCollection + */ + private Map imCols; + /** 转换器*/ + private List converters; + /** 转换器类型, 针对需要new的时候传入mapper参数的*/ + private List> converterClazz; + /** 编码*/ + private String encoding = "utf-8"; + /** */ + private String header = ""; + + public JsonSerializerXStreamImpl() { + + xs = new XStream(new JettisonMappedXmlDriver()); + xs.setMode(XStream.NO_REFERENCES); + // 打开标注 + xs.autodetectAnnotations(true); + + } + + public List getConverters() { + return converters; + } + + public void setConverters(List converters) { + this.converters = converters; + for(SingleValueConverter c : converters) { + xs.registerConverter(c); + } + } + + @Override + public byte[] Marshal(Object obj) throws SeriallizeException { + String h = MessageFormat.format(header, new Object[]{encoding}); + try { + byte[] objBs = xs.toXML(obj).getBytes(encoding); + byte[] r = new byte[h.getBytes().length+objBs.length]; + System.arraycopy(h.getBytes(), 0, r, 0, h.getBytes().length); + System.arraycopy(objBs, 0, r, h.getBytes().length, objBs.length); + return r; + } catch (UnsupportedEncodingException e) { + throw new SeriallizeException("Serializer Marshal Exception: "+e.getMessage()); + } + + } + + @Override + public Object Unmarshal(byte[] xml) throws SeriallizeException { + return xs.fromXML(new ByteArrayInputStream(xml)); + + } + + public Map getTypeAlias() { + return typeAlias; + } + + public void setTypeAlias(Map typeAlias) { + this.typeAlias = typeAlias; + // + for(String alias : typeAlias.keySet()) { + xs.alias(alias, typeAlias.get(alias)); + } + } + + public String getHeader() { + return header; + } + + public void setHeader(String header) { + this.header = header; + } + + public Map getImCols() { + return imCols; + } + + public void setImCols(Map imCols) { + this.imCols = imCols; + for(String imCol : imCols.keySet()) { + xs.addImplicitCollection(imCols.get(imCol), imCol); + } + + } + + /** + * @return the converterClazz + */ + public List> getConverterClazz() { + return converterClazz; + } + + /** + * @param converterClazz the converterClazz to set + */ + public void setConverterClazz(List> converterClazzs) { + for(Class cc : converterClazzs) { + Converter c; + try { + c = cc.getDeclaredConstructor(Mapper.class).newInstance(xs.getMapper()); + xs.registerConverter(c); + } catch (Exception e) { + + } + + } + + } + + /** + * @return the omitAlias + */ + public Map getOmitAlias() { + return omitAlias; + } + + /** + * @param omitAlias the omitAlias to set + */ + public void setOmitAlias(Map omitAlias) { + this.omitAlias = omitAlias; + } + + public String getEncoding() { + return encoding; + } + + public void setEncoding(String encoding) { + this.encoding = encoding; + } +} diff --git a/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/xstream/XmlSerializerXStreamImpl.java b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/xstream/XmlSerializerXStreamImpl.java new file mode 100644 index 00000000..cd0daacc --- /dev/null +++ b/rdb2graph/rdb2graph-common/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/common/serializer/xstream/XmlSerializerXStreamImpl.java @@ -0,0 +1,196 @@ +/** + * %commons% + * %1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.common.serializer.xstream; + +import java.io.ByteArrayInputStream; +import java.io.UnsupportedEncodingException; +import java.io.Writer; +import java.text.MessageFormat; +import java.util.List; +import java.util.Map; + +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.Serializer; +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.SeriallizeException; +import com.thoughtworks.xstream.XStream; +import com.thoughtworks.xstream.converters.Converter; +import com.thoughtworks.xstream.converters.SingleValueConverter; +import com.thoughtworks.xstream.core.util.QuickWriter; +import com.thoughtworks.xstream.io.HierarchicalStreamWriter; +import com.thoughtworks.xstream.io.xml.PrettyPrintWriter; +import com.thoughtworks.xstream.io.xml.StaxDriver; +import com.thoughtworks.xstream.mapper.Mapper; + +/** + * @类名: SerializerXStreamImpl + * @说明: 序列化器XStream实现 + * + * + * @author leehom + * @Date 2010-6-9 上午10:58:40 + * 修改记录： + * *. 支持CData标签 + * + * @see + */ +public class XmlSerializerXStreamImpl implements Serializer { + + /** 序列化库*/ + protected XStream xs; + /** 类型关联*/ + private Map typeAlias; + /** 忽略字段*/ + private Map omitAlias; + /** + * 取消标记, + * @ XStream.addImplicitCollection + */ + private Map imCols; + /** 转换器*/ + private List converters; + /** 转换器类型, 针对需要new的时候传入mapper参数的*/ + private List> converterClazz; + /** 编码*/ + private String encoding = "utf-8"; + /** xml文件头, {0}: endcoding*/ + private String header = ""; + + public XmlSerializerXStreamImpl() { + + xs = new XStream( + // StaxDriver支持namespace + // 支持 CData + new StaxDriver() { + public HierarchicalStreamWriter createWriter(Writer out) { + return new PrettyPrintWriter(out) { + protected void writeText(QuickWriter writer, String text) { + if(text.startsWith(PREFIX_CDATA) + && text.endsWith(SUFFIX_CDATA)) { + writer.write(text); + }else{ + super.writeText(writer, text); + } + } + }; + } + } + ); + + // 打开标注 + //xs.autodetectAnnotations(true); + + } + + public List getConverters() { + return converters; + } + + public void setConverters(List converters) { + this.converters = converters; + for(SingleValueConverter c : converters) { + xs.registerConverter(c); + } + } + + @Override + public byte[] Marshal(Object obj) throws SeriallizeException { + xs.setClassLoader(obj.getClass().getClassLoader()); + String h = MessageFormat.format(header, new Object[]{encoding}); + try { + byte[] objBs = xs.toXML(obj).getBytes(encoding); + byte[] r = new byte[h.getBytes().length+objBs.length]; + System.arraycopy(h.getBytes(), 0, r, 0, h.getBytes().length); + System.arraycopy(objBs, 0, r, h.getBytes().length, objBs.length); + return r; + } catch (UnsupportedEncodingException e) { + throw new SeriallizeException("Serializer Marshal Exception: "+e.getMessage()); + } + + } + + @Override + public Object Unmarshal(byte[] xml) throws SeriallizeException { + xs.setClassLoader(this.getClass().getClassLoader()); + return xs.fromXML(new ByteArrayInputStream(xml)); + + } + + public Map getTypeAlias() { + return typeAlias; + } + + public void setTypeAlias(Map typeAlias) { + this.typeAlias = typeAlias; + // + for(String alias : typeAlias.keySet()) { + xs.alias(alias, typeAlias.get(alias)); + } + } + + public String getHeader() { + return header; + } + + public void setHeader(String header) { + this.header = header; + } + + public Map getImCols() { + return imCols; + } + + public void setImCols(Map imCols) { + this.imCols = imCols; + for(String imCol : imCols.keySet()) { + xs.addImplicitCollection(imCols.get(imCol), imCol); + } + + } + + /** + * @return the converterClazz + */ + public List> getConverterClazz() { + return converterClazz; + } + + /** + * @ converterClazz the converterClazz to set + */ + public void setConverterClazz(List> converterClazzs) { + for(Class cc : converterClazzs) { + Converter c; + try { + c = cc.getDeclaredConstructor(Mapper.class).newInstance(xs.getMapper()); + xs.registerConverter(c); + } catch (Exception e) { + + } + + } + + } + + /** + * @return the omitAlias + */ + public Map getOmitAlias() { + return omitAlias; + } + + /** + * @param omitAlias the omitAlias to set + */ + public void setOmitAlias(Map omitAlias) { + this.omitAlias = omitAlias; + } + + public String getEncoding() { + return encoding; + } + + public void setEncoding(String encoding) { + this.encoding = encoding; + } +} diff --git a/rdb2graph/rdb2graph-datax/README.md b/rdb2graph/rdb2graph-datax/README.md new file mode 100644 index 00000000..e69de29b diff --git a/rdb2graph/rdb2graph-datax/pom.xml b/rdb2graph/rdb2graph-datax/pom.xml new file mode 100644 index 00000000..244e94e1 --- /dev/null +++ b/rdb2graph/rdb2graph-datax/pom.xml @@ -0,0 +1,49 @@ + + 4.0.0 + + com.leehom.arch.datax.plugin + rdb2graph-parent + ${revision} + + rdb2graph-datax + jar + + + 0.9.0 + UTF-8 + + + + + + com.google.guava + guava + + + org.projectlombok + lombok + + + + com.alibaba.datax + datax-core + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + logback.xml + application.yml + + + + + + diff --git a/rdb2graph/rdb2graph-datax/src/main/java/com/leehom/arch/datax/rdb2graph/EngineMain.java b/rdb2graph/rdb2graph-datax/src/main/java/com/leehom/arch/datax/rdb2graph/EngineMain.java new file mode 100644 index 00000000..e3c3d96d --- /dev/null +++ b/rdb2graph/rdb2graph-datax/src/main/java/com/leehom/arch/datax/rdb2graph/EngineMain.java @@ -0,0 +1,60 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.rdb2graph; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.datax.core.Engine; + +/** + * @类名: EngineMain + * @说明: datax engine主入口，执行两个engine + * 1. 行数据同步 + * 2. 关系同步 + * + * @author leehom + * @Date 2022年4月26日 下午6:56:23 + * 修改记录： + * + * @see + */ +public class EngineMain { + + public static final Logger log = LoggerFactory.getLogger(EngineMain.class); + + public static void main(String[] args) { + System.setProperty("datax.home", getCurrentClasspath()); + try { + log.info("行数据同步开始>>>"); + String[] dataxArgs1 = {"-job", getCurrentClasspath() + "/job/mysql/row2node.json", "-mode", "standalone", "-jobid", "1001"}; + // Engine.entry(dataxArgs1); + log.info("行数据同步完成."); + } catch (Throwable e) { + log.error(e.getMessage()); + } + log.info("关系同步开始>>>"); + try { + String[] dataxArgs2 = {"-job", getCurrentClasspath() + "/job/mysql/fk2rel.json", "-mode", "standalone", "-jobid", "1001"}; + Engine.entry(dataxArgs2); + log.info("关系同步完成."); + } catch (Throwable e) { + log.error(e.getMessage()); + } + } + + // + public static String getCurrentClasspath() { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + String currentClasspath = classLoader.getResource("").getPath(); + // 当前操作系统 + String osName = System.getProperty("os.name"); + if (osName.startsWith("Windows")) { + // 删除path中最前面的/ + currentClasspath = currentClasspath.substring(1); + } + return currentClasspath; + } +} diff --git a/rdb2graph/rdb2graph-datax/src/main/resources/conf/core.json b/rdb2graph/rdb2graph-datax/src/main/resources/conf/core.json new file mode 100644 index 00000000..473efb2d --- /dev/null +++ b/rdb2graph/rdb2graph-datax/src/main/resources/conf/core.json @@ -0,0 +1,65 @@ + +{ + "entry": { + "jvm": "-Xms4G -Xmx4G", + "environment": {} + }, + "common": { + "column": { + "datetimeFormat": "yyyy-MM-dd HH:mm:ss", + "timeFormat": "HH:mm:ss", + "dateFormat": "yyyy-MM-dd", + "extraFormats":["yyyyMMdd"], + "timeZone": "GMT+8", + "encoding": "utf-8" + } + }, + "core": { + "dataXServer": { + "address": "http://localhost:7001/api", + "timeout": 10000, + "reportDataxLog": false, + "reportPerfLog": false + }, + "transport": { + "channel": { + "class": "com.alibaba.datax.core.transport.channel.memory.MemoryChannel", + "speed": { + "byte": -1, + "record": -1 + }, + "flowControlInterval": 20, + "capacity": 512, + "byteCapacity": 67108864 + }, + "exchanger": { + "class": "com.alibaba.datax.core.plugin.BufferedRecordExchanger", + "bufferSize": 32 + } + }, + "container": { + "job": { + "reportInterval": 10000 + }, + "taskGroup": { + "channel": 5 + }, + "trace": { + "enable": "false" + } + + }, + "statistics": { + "collector": { + "plugin": { + "taskClass": "com.alibaba.datax.core.statistics.plugin.task.StdoutPluginCollector", + "maxDirtyNumber": 10 + } + } + }, + "metrics": { + "reporter": "nop", + "url": "192.168.1.25:9091" + } + } +} diff --git a/rdb2graph/rdb2graph-datax/src/main/resources/job/mysql/fk2rel.json b/rdb2graph/rdb2graph-datax/src/main/resources/job/mysql/fk2rel.json new file mode 100644 index 00000000..e131f9aa --- /dev/null +++ b/rdb2graph/rdb2graph-datax/src/main/resources/job/mysql/fk2rel.json @@ -0,0 +1,46 @@ +{ + "job": { + "setting": { + "speed": { + "channel": 7 + } + }, + "content": [ + { + "reader": { + "name": "mysqlreader4graph", + "parameter": { + "username": "root", + "password": "123456", + "phase": "rel", + "schemaUri": "sakila.xml", + "column": ["*"], + "connection": [ + { + "querySql": [ + + ], + "jdbcUrl": [ + "jdbc:mysql://localhost:3306/sakila?remarks=true&useInformationSchema=false&serverTimezone=Asia/Shanghai" + ] + } + ] + } + }, + "writer": { + "name": "neo4jwriter", + "parameter": { + "username": "neo4j", + "password": "neo4j1234", + "phase": "rel", + "database": "sakila", + "batchSize": 100, + "uri": "bolt://192.168.1.20:7687", + "schemaUri": "sakila.xml", + "writemode": "insert" + } + } + } + ] + } +} \ No newline at end of file diff --git a/rdb2graph/rdb2graph-datax/src/main/resources/job/mysql/row2node.json b/rdb2graph/rdb2graph-datax/src/main/resources/job/mysql/row2node.json new file mode 100644 index 00000000..4c5a10a2 --- /dev/null +++ b/rdb2graph/rdb2graph-datax/src/main/resources/job/mysql/row2node.json @@ -0,0 +1,46 @@ +{ + "job": { + "setting": { + "speed": { + "channel": 5 + } + }, + "content": [ + { + "reader": { + "name": "mysqlreader4graph", + "parameter": { + "username": "root", + "password": "123456", + "phase": "table", + "schemaUri": "sakila.xml", + "column": ["*"], + "connection": [ + { + "table": [ + + ], + "jdbcUrl": [ + "jdbc:mysql://localhost:3306/sakila?remarks=true&useInformationSchema=false&serverTimezone=Asia/Shanghai" + ] + } + ] + } + }, + "writer": { + "name": "neo4jwriter", + "parameter": { + "username": "neo4j", + "password": "neo4j1234", + "phase": "table", + "database": "sakila", + "batchSize": 100, + "uri": "bolt://192.168.1.20:7687", + "schemaUri": "sakila.xml", + "writemode": "insert" + } + } + } + ] + } +} \ No newline at end of file diff --git a/rdb2graph/rdb2graph-datax/src/main/resources/logback.xml b/rdb2graph/rdb2graph-datax/src/main/resources/logback.xml new file mode 100644 index 00000000..0f648972 --- /dev/null +++ b/rdb2graph/rdb2graph-datax/src/main/resources/logback.xml @@ -0,0 +1,76 @@ + + + + spring-boot-log + + + + + + [%date] %clr([%level]) %clr([%logger]:%L) >>> %msg %n + utf-8 + + + + INFO + + + + + logs/info.log + + + + logs/info-%d{yyyy-MM-dd}.%i.log + + 128MB + + + 30 + + true + + [%date] %clr([%level]) %clr([%logger]:%L) >>> %msg %n + utf-8 + + + info + ACCEPT + DENY + + + + + logs/error.log + + + + logs/error-%d{yyyy-MM-dd}.%i.log + + 2MB + + + 180 + + true + + + [%date] %clr([%level]) [%thread] %clr([%logger]:%L) >>> %msg %n + utf-8 + + + + + ERROR + + ACCEPT + + DENY + + + + + + + + diff --git a/rdb2graph/rdb2graph-datax/src/main/resources/rdb2graph.json.template b/rdb2graph/rdb2graph-datax/src/main/resources/rdb2graph.json.template new file mode 100644 index 00000000..868e95f6 --- /dev/null +++ b/rdb2graph/rdb2graph-datax/src/main/resources/rdb2graph.json.template @@ -0,0 +1,57 @@ +{ + "job": { + "setting": { + "speed": { + "channel": 5 + } + }, + "content": [ + { + "reader": { + "name": "oraclereader4graph", + "phase": "table" // rel + "parameter": { + "username": "root", + "password": "root", + //*******多url支持分库分表****** + "connection": [ + { + "table": [ + "table" + ], + "jdbcUrl": [ + "jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]" + ] + } + ] 或 + "connection": [ + { + "querySql": [ + + ], + "relFrom": [], + "relFk": [], + "jdbcUrl": [ + "jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]" + ] + } + ] + //**************** + } + }, + "writer": { + "name": "neo4jwriter", + "parameter": { + "username": "root", + "password": "root", + "phase": # table/rel + "database": + "uri": "", + "schemaUri": "" + "writemode": "insert" // replace，update + } + } + } + ] + } +} \ No newline at end of file diff --git a/rdb2graph/rdb2graph-datax/src/main/resources/sakila.xml b/rdb2graph/rdb2graph-datax/src/main/resources/sakila.xml new file mode 100644 index 00000000..6c567ab7 --- /dev/null +++ b/rdb2graph/rdb2graph-datax/src/main/resources/sakila.xml @@ -0,0 +1,1942 @@ + + sakila + + + actor + + + actor_id + SMALLINT + 5 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + ACTOR_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_actor_last_name + OTHER + + + false +
+ + address + + + address_id + SMALLINT + 5 + + + + address + VARCHAR + 50 + + + + address2 + VARCHAR + 50 + + + + district + VARCHAR + 20 + + + + city_id + SMALLINT + 5 + + + + postal_code + VARCHAR + 10 + + + + phone + VARCHAR + 20 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + ADDRESS_ID_NN + + + + + ADDRESS_NN + + + + + ADDRESS2_NN + + + + + DISTRICT_NN + + + + + CITY_ID_NN + + + + + POSTAL_CODE_NN + + + + + PHONE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + + fk_address_city_idx + OTHER + + + + + + + idx_fk_city_id + OTHER + + + false +
+ + category + + + category_id + TINYINT + 3 + + + + name + VARCHAR + 25 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + CATEGORY_ID_NN + + + + + NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false +
+ + city + + + city_id + SMALLINT + 5 + + + + city + VARCHAR + 50 + + + + country_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_city_country + + country + + + country_id + SMALLINT + 5 + + + + country + VARCHAR + 50 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + COUNTRY_ID_NN + + + + + COUNTRY_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false + + + + + + + + + + + + + + CITY_ID_NN + + + + + CITY_NN + + + + + COUNTRY_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_country_id + OTHER + + + false +
+ +
+ customer + + + customer_id + SMALLINT + 5 + + + + store_id + TINYINT + 3 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + email + VARCHAR + 50 + + + + address_id + SMALLINT + 5 + + + + active + BIT + 1 + + + + create_date + TIMESTAMP + 19 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_customer_address + + + + + + + + + + + + + + fk_customer_store + + store + + + store_id + TINYINT + 3 + + + + manager_staff_id + TINYINT + 3 + + + + address_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_store_address + + + + + + + + + + + + + + fk_store_staff + + staff + + + staff_id + TINYINT + 3 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + address_id + SMALLINT + 5 + + + + picture + LONGVARBINARY + 65535 + + + + email + VARCHAR + 50 + + + + store_id + TINYINT + 3 + + + + active + BIT + 1 + + + + username + VARCHAR + 16 + + + + password + VARCHAR + 40 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_staff_address + + + + + + + + + + + + + + fk_staff_store + + + + + + + + + + + + + + STAFF_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + ADDRESS_ID_NN + + + + + PICTURE_NN + + + + + EMAIL_NN + + + + + STORE_ID_NN + + + + + ACTIVE_NN + + + + + USERNAME_NN + + + + + PASSWORD_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + + + + + idx_fk_store_id + OTHER + + + false + + + + + + + + + + + + + + STORE_ID_NN + + + + + MANAGER_STAFF_ID_NN + + + + + ADDRESS_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + idx_unique_manager + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + false + + + + + + + + + + + + + + CUSTOMER_ID_NN + + + + + STORE_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + EMAIL_NN + + + + + ADDRESS_ID_NN + + + + + ACTIVE_NN + + + + + CREATE_DATE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + + + + + idx_fk_store_id + OTHER + + + + + + + idx_last_name + OTHER + + + false +
+ + film + + + film_id + SMALLINT + 5 + + + + title + VARCHAR + 255 + + + + description + LONGVARCHAR + 65535 + + + + release_year + DATE + 4 + + + + language_id + TINYINT + 3 + + + + original_language_id + TINYINT + 3 + + + + rental_duration + TINYINT + 3 + + + + rental_rate + DECIMAL + 4 + + + + length + SMALLINT + 5 + + + + replacement_cost + DECIMAL + 5 + + + + rating + CHAR + 5 + + + + special_features + CHAR + 54 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_film_language + + language + + + language_id + TINYINT + 3 + + + + name + CHAR + 20 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + LANGUAGE_ID_NN + + + + + NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false + + + + + + + + + + + + + + fk_film_language_original + + + + + + + + + + + + + + FILM_ID_NN + + + + + TITLE_NN + + + + + DESCRIPTION_NN + + + + + RELEASE_YEAR_NN + + + + + LANGUAGE_ID_NN + + + + + ORIGINAL_LANGUAGE_ID_NN + + + + + RENTAL_DURATION_NN + + + + + RENTAL_RATE_NN + + + + + LENGTH_NN + + + + + REPLACEMENT_COST_NN + + + + + RATING_NN + + + + + SPECIAL_FEATURES_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_language_id + OTHER + + + + + + + idx_fk_original_language_id + OTHER + + + + + + + idx_title + OTHER + + + false +
+ + film_actor + + + actor_id + SMALLINT + 5 + + + + film_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + + fk_film_actor_actor + + + + + + + + + + + + + + fk_film_actor_film + + + + + + + + + + + + + + ACTOR_ID_NN + + + + + FILM_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + PRIMARY + + + + + + + + + idx_fk_film_id + OTHER + + + true + +
+ + film_category + + + film_id + SMALLINT + 5 + + + + category_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + + fk_film_category_category + + + + + + + + + + + + + + fk_film_category_film + + + + + + + + + + + + + + FILM_ID_NN + + + + + CATEGORY_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + PRIMARY + + + + + + + + + fk_film_category_category + OTHER + + + true + +
+ + film_text + + + film_id + SMALLINT + 5 + + + + title + VARCHAR + 255 + + + + description + LONGVARCHAR + 65535 + + + + + + + + + + + + + + + FILM_ID_NN + + + + + TITLE_NN + + + + + DESCRIPTION_NN + + + + + + + + + PRIMARY + + + + + + + + + + idx_title_description + OTHER + + + false +
+ + inventory + + + inventory_id + INTEGER + 8 + + + + film_id + SMALLINT + 5 + + + + store_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_inventory_film + + + + + + + + + + + + + + fk_inventory_store + + + + + + + + + + + + + + INVENTORY_ID_NN + + + + + FILM_ID_NN + + + + + STORE_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_film_id + OTHER + + + + + + + + idx_store_id_film_id + OTHER + + + false +
+ +
+ payment + + + payment_id + SMALLINT + 5 + + + + customer_id + SMALLINT + 5 + + + + staff_id + TINYINT + 3 + + + + rental_id + INTEGER + 10 + + + + amount + DECIMAL + 5 + + + + payment_date + TIMESTAMP + 19 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_payment_customer + + + + + + + + + + + + + + fk_payment_rental + + rental + + + rental_id + INTEGER + 10 + + + + rental_date + TIMESTAMP + 19 + + + + inventory_id + INTEGER + 8 + + + + customer_id + SMALLINT + 5 + + + + return_date + TIMESTAMP + 19 + + + + staff_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_rental_customer + + + + + + + + + + + + + + fk_rental_inventory + + + + + + + + + + + + + + fk_rental_staff + + + + + + + + + + + + + + RENTAL_ID_NN + + + + + RENTAL_DATE_NN + + + + + INVENTORY_ID_NN + + + + + CUSTOMER_ID_NN + + + + + RETURN_DATE_NN + + + + + STAFF_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + rental_date + + + + + + + rental_date + + + + + + + rental_date + + + + + + + + + idx_fk_customer_id + OTHER + + + + + + + idx_fk_inventory_id + OTHER + + + + + + + idx_fk_staff_id + OTHER + + + false + + + + + + + + + + + + + + fk_payment_staff + + + + + + + + + + + + + + PAYMENT_ID_NN + + + + + CUSTOMER_ID_NN + + + + + STAFF_ID_NN + + + + + RENTAL_ID_NN + + + + + AMOUNT_NN + + + + + PAYMENT_DATE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + fk_payment_rental + OTHER + + + + + + + idx_fk_customer_id + OTHER + + + + + + + idx_fk_staff_id + OTHER + + + false +
+ +
+
+ + \ No newline at end of file diff --git a/rdb2graph/rdb2graph-scanner/pom.xml b/rdb2graph/rdb2graph-scanner/pom.xml new file mode 100644 index 00000000..eb0ff339 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/pom.xml @@ -0,0 +1,74 @@ + + + 4.0.0 + + com.leehom.arch.datax.plugin + rdb2graph-parent + ${revision} + + rdb2graph-scanner + jar + + + + 4.0.2 + + + + + com.alibaba + druid + + + mysql + mysql-connector-java + ${mysql.driver.version} + + + + org.neo4j.driver + neo4j-java-driver + ${neo4j-java-driver.version} + + + + org.projectlombok + lombok + true + + + + com.google.guava + guava + + + com.oracle + ojdbc6 + 11.2.0.3 + system + ${basedir}/src/main/lib/ojdbc6-11.2.0.3.jar + + + + com.leehom.arch.datax.plugin + rdb2graph-common + ${project.version} + + + + junit + junit + test + + + + com.alibaba + fastjson + test + + + + + + diff --git a/rdb2graph/rdb2graph-scanner/sakila.xml b/rdb2graph/rdb2graph-scanner/sakila.xml new file mode 100644 index 00000000..65ef1eaa --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/sakila.xml @@ -0,0 +1,1849 @@ + + sakila + +
+ actor + + + actor_id + SMALLINT + 5 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + ACTOR_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_actor_last_name + OTHER + + + false +
+ + address + + + address_id + SMALLINT + 5 + + + + address + VARCHAR + 50 + + + + address2 + VARCHAR + 50 + + + + district + VARCHAR + 20 + + + + city_id + SMALLINT + 5 + + + + postal_code + VARCHAR + 10 + + + + phone + VARCHAR + 20 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_address_city + + city + + + city_id + SMALLINT + 5 + + + + city + VARCHAR + 50 + + + + country_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_city_country + + country + + + country_id + SMALLINT + 5 + + + + country + VARCHAR + 50 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + COUNTRY_ID_NN + + + + + COUNTRY_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false + + + + + + + + + + + + + + CITY_ID_NN + + + + + CITY_NN + + + + + COUNTRY_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_country_id + OTHER + + + false + + + + + + + + + + + + + + ADDRESS_ID_NN + + + + + ADDRESS_NN + + + + + DISTRICT_NN + + + + + CITY_ID_NN + + + + + PHONE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_city_id + OTHER + + + false +
+ + category + + + category_id + TINYINT + 3 + + + + name + VARCHAR + 25 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + CATEGORY_ID_NN + + + + + NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false +
+ +
+
+ customer + + + customer_id + SMALLINT + 5 + + + + store_id + TINYINT + 3 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + email + VARCHAR + 50 + + + + address_id + SMALLINT + 5 + + + + active + BIT + 0 + + + + create_date + TIMESTAMP + 19 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_customer_address + + + + + + + + + + + + + + fk_customer_store + + store + + + store_id + TINYINT + 3 + + + + manager_staff_id + TINYINT + 3 + + + + address_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_store_address + + + + + + + + + + + + + + fk_store_staff + + staff + + + staff_id + TINYINT + 3 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + address_id + SMALLINT + 5 + + + + picture + LONGVARBINARY + 65535 + + + + email + VARCHAR + 50 + + + + store_id + TINYINT + 3 + + + + active + BIT + 0 + + + + username + VARCHAR + 16 + + + + password + VARCHAR + 40 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_staff_address + + + + + + + + + + + + + + fk_staff_store + + + + + + + + + + + + + + STAFF_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + ADDRESS_ID_NN + + + + + STORE_ID_NN + + + + + ACTIVE_NN + + + + + USERNAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + + + + + idx_fk_store_id + OTHER + + + false + + + + + + + + + + + + + + STORE_ID_NN + + + + + MANAGER_STAFF_ID_NN + + + + + ADDRESS_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + idx_unique_manager + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + false + + + + + + + + + + + + + + CUSTOMER_ID_NN + + + + + STORE_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + ADDRESS_ID_NN + + + + + ACTIVE_NN + + + + + CREATE_DATE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + + + + + idx_fk_store_id + OTHER + + + + + + + idx_last_name + OTHER + + + false +
+ + film + + + film_id + SMALLINT + 5 + + + + title + VARCHAR + 128 + + + + description + LONGVARCHAR + 65535 + + + + release_year + DATE + 0 + + + + language_id + TINYINT + 3 + + + + original_language_id + TINYINT + 3 + + + + rental_duration + TINYINT + 3 + + + + rental_rate + DECIMAL + 4 + + + + length + SMALLINT + 5 + + + + replacement_cost + DECIMAL + 5 + + + + rating + CHAR + 5 + + + + special_features + CHAR + 54 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_film_language + + language + + + language_id + TINYINT + 3 + + + + name + CHAR + 20 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + LANGUAGE_ID_NN + + + + + NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false + + + + + + + + + + + + + + fk_film_language_original + + + + + + + + + + + + + + FILM_ID_NN + + + + + TITLE_NN + + + + + LANGUAGE_ID_NN + + + + + RENTAL_DURATION_NN + + + + + RENTAL_RATE_NN + + + + + REPLACEMENT_COST_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_language_id + OTHER + + + + + + + idx_fk_original_language_id + OTHER + + + + + + + idx_title + OTHER + + + false +
+ + film_actor + + + actor_id + SMALLINT + 5 + + + + film_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + + fk_film_actor_actor + + + + + + + + + + + + + + fk_film_actor_film + + + + + + + + + + + + + + ACTOR_ID_NN + + + + + FILM_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_film_id + OTHER + + + true + +
+ + film_category + + + film_id + SMALLINT + 5 + + + + category_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + + fk_film_category_category + + + + + + + + + + + + + + fk_film_category_film + + + + + + + + + + + + + + FILM_ID_NN + + + + + CATEGORY_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + + PRIMARY + + + + + + + + + fk_film_category_category + OTHER + + + true + +
+ + film_text + + + film_id + SMALLINT + 5 + + + + title + VARCHAR + 255 + + + + description + LONGVARCHAR + 65535 + + + + + + + + + + + + + + + FILM_ID_NN + + + + + TITLE_NN + + + + + + + + + PRIMARY + + + + + + + + + + idx_title_description + OTHER + + + false +
+ + inventory + + + inventory_id + INTEGER + 8 + + + + film_id + SMALLINT + 5 + + + + store_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_inventory_film + + + + + + + + + + + + + + fk_inventory_store + + + + + + + + + + + + + + INVENTORY_ID_NN + + + + + FILM_ID_NN + + + + + STORE_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_film_id + OTHER + + + + + + + + idx_store_id_film_id + OTHER + + + false +
+ +
+ payment + + + payment_id + SMALLINT + 5 + + + + customer_id + SMALLINT + 5 + + + + staff_id + TINYINT + 3 + + + + rental_id + INTEGER + 10 + + + + amount + DECIMAL + 5 + + + + payment_date + TIMESTAMP + 19 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_payment_customer + + + + + + + + + + + + + + fk_payment_rental + + rental + + + rental_id + INTEGER + 10 + + + + rental_date + TIMESTAMP + 19 + + + + inventory_id + INTEGER + 8 + + + + customer_id + SMALLINT + 5 + + + + return_date + TIMESTAMP + 19 + + + + staff_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_rental_customer + + + + + + + + + + + + + + fk_rental_inventory + + + + + + + + + + + + + + fk_rental_staff + + + + + + + + + + + + + + RENTAL_ID_NN + + + + + RENTAL_DATE_NN + + + + + INVENTORY_ID_NN + + + + + CUSTOMER_ID_NN + + + + + STAFF_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + rental_date + + + + + + + + + idx_fk_customer_id + OTHER + + + + + + + idx_fk_inventory_id + OTHER + + + + + + + idx_fk_staff_id + OTHER + + + false + + + + + + + + + + + + + + fk_payment_staff + + + + + + + + + + + + + + PAYMENT_ID_NN + + + + + CUSTOMER_ID_NN + + + + + STAFF_ID_NN + + + + + AMOUNT_NN + + + + + PAYMENT_DATE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + fk_payment_rental + OTHER + + + + + + + idx_fk_customer_id + OTHER + + + + + + + idx_fk_staff_id + OTHER + + + false +
+ +
+
+ + \ No newline at end of file diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/AbstractDbScanner.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/AbstractDbScanner.java new file mode 100644 index 00000000..54b87dfa --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/AbstractDbScanner.java @@ -0,0 +1,75 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner; + +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.NotNullConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.PKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.UniqueConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.BaseDataSourceBean; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.DruidConnectionProperties; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.IndexMetadata; + +import lombok.Data; + +/** + * @类名: AbstractDbScanner + * @说明: 抽象库扫描, + * 获取schema，包括表，字段，索引，约束 + * + * @author leehom + * @Date 2022年1月7日 下午2:21:00 + * 修改记录： + * + * @see + */ +@Data +public abstract class AbstractDbScanner { + + protected DruidConnectionProperties connProps; + + // 扫描数据库模式 + public abstract DbSchema scan(BaseDataSourceBean dsBean) throws Exception; + // 扫描数据库 + protected abstract DbSchema doScanDb(DatabaseMetaData dbmd, ResultSet dbrs) throws SQLException; + // 扫描数据表 + protected abstract TableMetadata doScanTable(String dbName, DatabaseMetaData metadata, ResultSet tablers) throws SQLException; + // 扫描表的索引和约束 + // 主键/外键/非空/唯一 + protected abstract Optional doScanPK(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException; + // 扫描外键，在扫描表后 + protected abstract Map> doScanFK(String dbName, DatabaseMetaData dbmd, List tbmds) throws SQLException; + protected abstract List doScanNN(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException; + protected abstract List doScanUnique(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException; + // 索引 + protected abstract List doScanIndex(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException; + + // 表扫描过滤 + protected boolean acceptTable(String tableName) { + List tbs = connProps.getTbs(); + if((tbs==null||tbs.size()==0)||tbs.contains(tableName)) + return true; + return false; + } + // 查找外键引用表 + protected Optional findRefTable(String refName, List tbmds) { + for(TableMetadata tbmd: tbmds) { + if(refName.equals(tbmd.getName())) { + return Optional.of(tbmd); + } + } + return Optional.empty(); + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/MysqlScanner.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/MysqlScanner.java new file mode 100644 index 00000000..6b9927be --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/MysqlScanner.java @@ -0,0 +1,322 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.JDBCType; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import javax.sql.DataSource; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchemaConsts; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.NotNullConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.PKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.UniqueConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKField; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.BaseDataSourceBean; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.IndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.IndexType; + +/** + * @类名: MysqlScanner + * @说明: 数据库扫描, + * 表，字段，索引，约束 + * + * @author leehom + * @Date 2022年1月7日 下午2:21:00 + * 修改记录： + * + * @see + */ +public class MysqlScanner extends AbstractDbScanner { + + // 扫描数据库模式 + public DbSchema scan(BaseDataSourceBean dsBean) throws Exception { + // 构建数据源，获取数据库连接 + DataSource ds = dsBean.toDataSource(); + Connection conn = ds.getConnection(); + // + String db = connProps.getDb(); + // db元数据 + DatabaseMetaData dbmd = conn.getMetaData(); + // 模式结果集 + ResultSet schemars = dbmd.getCatalogs(); + // + while (schemars.next()) { + // 扫描数据库 + String dbName = schemars.getString(DbSchemaConsts.DB_NAME_INDEX); + if(db.equals(dbName)) { + DbSchema schema = this.doScanDb(dbmd, schemars); + schemars.close(); + return schema; + } + } + // TODO 使用统一异常设计 + throw new Exception("no db found!"); + + } + + // 扫描数据库 + @Override + public DbSchema doScanDb(DatabaseMetaData metadata, ResultSet dbrs) throws SQLException { + // + DbSchema dbmd = new DbSchema(); + // + String dbName = dbrs.getString(DbSchemaConsts.DB_NAME_INDEX); + dbmd.setName(dbName); + // 表元数据, 只获取表 + ResultSet tablers = metadata.getTables(dbName, null, null, new String[] {DbSchemaConsts.TABLE_TYPE}); + List tbmds = new ArrayList<>(); + while (tablers.next()) { + // 扫描表 + String tbName = tablers.getString(DbSchemaConsts.TABLE_NAME_INDEX); + if(!this.acceptTable(tbName)) + continue; + TableMetadata tbmd = this.doScanTable(dbName, metadata, tablers); + tbmds.add(tbmd); + + } + dbmd.setTables(tbmds); + tablers.close(); + // 表扫描完，扫描外键, 外键需引用其他表 + doScanFK(dbName, metadata, tbmds); + // + return dbmd; + } + + // 扫描数据表 + @Override + public TableMetadata doScanTable(String dbName, DatabaseMetaData metadata, ResultSet tablers) throws SQLException { + TableMetadata tbmd = new TableMetadata(); + String tbName = tablers.getString(DbSchemaConsts.TABLE_NAME_INDEX); + tbmd.setName(tbName); + // 扫描表字段 + List fdmds = new ArrayList<>(); + ResultSet feildrs = metadata.getColumns(dbName, null, tbName, null); + // 用于调试 + // ResultSetMetaData fmd = feildrs.getMetaData(); + while (feildrs.next()) { + FieldMetadata fdmd = new FieldMetadata(); + // 字段名称/类型/注释 + String fieldName = feildrs.getString(DbSchemaConsts.FEILD_NAME_INDEX); + fdmd.setName(fieldName); + int type = feildrs.getInt(DbSchemaConsts.FEILD_DATA_TYPE_INDEX); + fdmd.setType(JDBCType.valueOf(type)); + int length = feildrs.getInt(DbSchemaConsts.FEILD_LENGTH_INDEX); + fdmd.setLength(length); + String remark = feildrs.getString(DbSchemaConsts.FEILD_REMARK_INDEX); + fdmd.setRemark(remark); + fdmds.add(fdmd); + } + feildrs.close(); + tbmd.setFields(fdmds); + // 扫描主键 + doScanPK(dbName, metadata, tbmd); + // 非空约束 + doScanNN(dbName, metadata, tbmd); + // 唯一约束 + doScanUnique(dbName, metadata, tbmd); + // 索引 + doScanIndex(dbName, metadata, tbmd); + // + return tbmd; + } + + // 扫描约束，主键；外键；非空；唯一 + @Override + protected Optional doScanPK(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException { + // 主键结果集 + ResultSet pkrs = dbmd.getPrimaryKeys(dbName, null, tbmd.getName()); + // 主键字段 + List pkfs = new ArrayList<>(); + // 主键元数据 + // ResultSetMetaData pkmd = pkrs.getMetaData(); + while(pkrs.next()){ + String pkFn = pkrs.getString(DbSchemaConsts.PK_NAME_INDEX); + Optional op = tbmd.findField(pkFn); + if(op.isPresent()) + pkfs.add(op.get()); + } + // 无主键 + if(pkfs.size()==0) { + return Optional.empty(); + } + tbmd.setPk(new PKConstraintMetadata(tbmd, pkfs)); + return Optional.of(tbmd.getPk()); + } + + @Override + protected Map> doScanFK(String dbName, DatabaseMetaData dbmd, List tbmds) throws SQLException { + Map> r = new HashMap<>(); + // 循环扫描表 + for(TableMetadata tbmd: tbmds) { + List fks = scanFKTable(dbName, dbmd, tbmd, tbmds); + r.put(tbmd.getName(), fks); + } + return r; + } + + /** + * @说明：扫描表外键索引 + * 一个表可有多个外键，一个外键可有多个字段 + * + * @author leehom + * @param dbName + * @param dbmd + * @param tbmd 扫描目标表 + * @param tbmds 所有表 + * @return + * @throws SQLException + * + */ + private List scanFKTable(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd, List tbmds) throws SQLException { + // 外键元数据结果集 + ResultSet fkrs = dbmd.getImportedKeys(dbName, dbName, tbmd.getName()); + // 主键元数据 + // ResultSetMetaData fkmd = fkrs.getMetaData(); + // + List fkcmds = new ArrayList<>(); + // key: 外键名称 + Map fkcmdMap = new HashMap<>(); + while(fkrs.next()){ + // 外键名称 + String fkName = fkrs.getString(DbSchemaConsts.FK_NAME_INDEX); + // 外键引用表 + String fkTableName = fkrs.getString(DbSchemaConsts.FK_REF_TABLE_INDEX); + TableMetadata refTable = this.findRefTable(fkTableName, tbmds).get(); + // 主表外键字段 + String pfn = fkrs.getString(DbSchemaConsts.FK_PK_FIELD_INDEX); + FieldMetadata pf = tbmd.findField(pfn).get(); + // + String ffn = fkrs.getString(DbSchemaConsts.FK_REF_FIELD_INDEX); + FieldMetadata ff = refTable.findField(ffn).get(); + // + FKField fkf = new FKField(pf, ff); + // 外键存在，即，多字段外键 + if(fkcmdMap.containsKey(fkName)) { + FKConstraintMetadata fkcmd = fkcmdMap.get(fkName); + fkcmd.getFields().add(pf); + fkcmd.getFkFields().add(fkf); + } else { + List fields = new ArrayList<>(); + fields.add(pf); + List fkFields = new ArrayList<>(); + fkFields.add(fkf); + FKConstraintMetadata fkcmd = new FKConstraintMetadata(fkName, tbmd, refTable, fields, fkFields); + fkcmds.add(fkcmd); + fkcmdMap.put(fkName, fkcmd); + } + } + // 外键放入表 + tbmd.setFks(fkcmds); + return fkcmds; + } + + @Override + protected List doScanNN(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException { + // + List nns = new ArrayList<>(); + // 字段元数据 + ResultSet feildrs = dbmd.getColumns(dbName, null, tbmd.getName(), null); + while (feildrs.next()) { + // + String fieldName = feildrs.getString(DbSchemaConsts.FEILD_NAME_INDEX); + int nullable = feildrs.getInt(DbSchemaConsts.FEILD_NULLABLE_INDEX); + // nullable==0 0为法false，即不能为空 + if(nullable==0) { + Optional op = tbmd.findField(fieldName); + if(op.isPresent()) { + nns.add(new NotNullConstraintMetadata(tbmd, op.get())); + } + } + } + feildrs.close(); + tbmd.setNotNull(nns); + return nns; + } + + // 唯一约束 + @Override + protected List doScanUnique(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException { + List uqmds = new ArrayList<>(); + // + ResultSet indexrs = dbmd.getIndexInfo(dbName, null, tbmd.getName(), false, false); + // ResultSetMetaData indexmd = indexrs.getMetaData(); + // + Map uqmdMap = new HashMap<>(); + while (indexrs.next()) { + // 名称，与索引共用 + String unqName = indexrs.getString(DbSchemaConsts.INDEX_NAME_INDEX); + // nonUnique=true, 即不唯一 + boolean nonUnique = indexrs.getBoolean(DbSchemaConsts.UNIQUE_INDEX); + if (nonUnique) + continue; + String fn = indexrs.getString(DbSchemaConsts.UNIQUE_FIELD_INDEX); + FieldMetadata field = tbmd.findField(fn).get(); + if(uqmdMap.containsKey(unqName)) { + UniqueConstraintMetadata uqmd = uqmdMap.get(unqName); + uqmd.getFields().add(field); + } else { + List fields = new ArrayList<>(); + fields.add(field); + UniqueConstraintMetadata uqmd = new UniqueConstraintMetadata(unqName, tbmd, fields); + uqmds.add(uqmd); + uqmdMap.put(unqName, uqmd); + } + } + tbmd.setUnique(uqmds); + return uqmds; + } + + // 索引 + @Override + protected List doScanIndex(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException { + List indexmds = new ArrayList<>(); + // + ResultSet indexrs = dbmd.getIndexInfo(dbName, null, tbmd.getName(), false, false); + // + // ResultSetMetaData indexrsmd = indexrs.getMetaData(); + // key: 索引名称 + Map indexmdMap = new HashMap<>(); + while (indexrs.next()) { + // + String indexName = indexrs.getString(DbSchemaConsts.INDEX_NAME_INDEX); + // 非空索引不处理，以非空约束处理 + boolean nonUnique = indexrs.getBoolean(DbSchemaConsts.UNIQUE_INDEX); + if (!nonUnique) + continue; + // 索引字段 + String fn = indexrs.getString(DbSchemaConsts.INDEX_FIELD_INDEX); + FieldMetadata field = tbmd.findField(fn).get(); + // 已有索引存储，多字段索引 + if(indexmdMap.containsKey(indexName)) { + IndexMetadata indexmd = indexmdMap.get(indexName); + indexmd.getFields().add(field); + } else { + List fields = new ArrayList<>(); + fields.add(field); + // 索引类型 + int indexType = indexrs.getInt(DbSchemaConsts.INDEX_TYPE_INDEX); + IndexMetadata indexmd = new IndexMetadata(indexName, tbmd, fields, IndexType.values()[indexType]); + indexmds.add(indexmd); + indexmdMap.put(indexName, indexmd); + } + } + tbmd.setIndexes(indexmds); + return indexmds; + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/OracleScanner.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/OracleScanner.java new file mode 100644 index 00000000..04bfbfbb --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/OracleScanner.java @@ -0,0 +1,308 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.JDBCType; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import javax.sql.DataSource; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchemaConsts; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.NotNullConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.PKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.UniqueConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKField; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.BaseDataSourceBean; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.IndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.IndexType; + +/** + * @类名: OracleScanner + * @说明: oracle数据库扫描, + * 表，字段，索引，约束 + * + * @author leehom + * @Date 2022年1月7日 下午2:21:00 + * 修改记录： + * + * @see + */ +public class OracleScanner extends AbstractDbScanner { + + // 扫描数据库模式 + public DbSchema scan(BaseDataSourceBean dsBean) throws Exception { + // 构建数据源，获取数据库连接 + DataSource ds = dsBean.toDataSource(); + Connection conn = ds.getConnection(); + String db = connProps.getDb(); + // db元数据 + DatabaseMetaData dbmd = conn.getMetaData(); + // 模式结果集 + ResultSet schemars = dbmd.getSchemas(); // dbmd.getCatalogs(); + // + while (schemars.next()) { + // 扫描数据库 + String dbName = schemars.getString(DbSchemaConsts.DB_NAME_INDEX); + if(db.equals(dbName)) { + DbSchema schema = this.doScanDb(dbmd, schemars); + schemars.close(); + return schema; + } + } + throw new Exception("no db found!"); + + } + + // 扫描数据库 + @Override + public DbSchema doScanDb(DatabaseMetaData metadata, ResultSet dbrs) throws SQLException { + // + DbSchema dbmd = new DbSchema(); + // + String dbName = dbrs.getString(DbSchemaConsts.DB_NAME_INDEX); + dbmd.setName(dbName); + // 表元数据 + ResultSet tablers = metadata.getTables(null, dbName, null, null); + List tbmds = new ArrayList<>(); + while (tablers.next()) { + // 扫描表 + TableMetadata tbmd = this.doScanTable(dbName, metadata, tablers); + tbmds.add(tbmd); + } + dbmd.setTables(tbmds); + tablers.close(); + // 扫描外键 + doScanFK(dbName, metadata, tbmds); + // + return dbmd; + } + + // 扫描数据表 + @Override + public TableMetadata doScanTable(String dbName, DatabaseMetaData metadata, ResultSet tablers) throws SQLException { + TableMetadata tbmd = new TableMetadata(); + String tbName = tablers.getString(DbSchemaConsts.TABLE_NAME_INDEX); + tbmd.setName(tbName); + // 扫描表字段 + List fdmds = new ArrayList<>(); + ResultSet feildrs = metadata.getColumns(null, dbName, tbName, null); + // + ResultSetMetaData fmd = feildrs.getMetaData(); + while (feildrs.next()) { + FieldMetadata fdmd = new FieldMetadata(); + // 字段名称/类型/注释 + String fieldName = feildrs.getString(DbSchemaConsts.FEILD_NAME_INDEX); + fdmd.setName(fieldName); + int type = feildrs.getInt(DbSchemaConsts.FEILD_DATA_TYPE_INDEX); + fdmd.setType(JDBCType.valueOf(type)); + int length = feildrs.getInt(DbSchemaConsts.FEILD_LENGTH_INDEX); + fdmd.setLength(length); + String remark = feildrs.getString(DbSchemaConsts.FEILD_REMARK_INDEX); + fdmd.setRemark(remark); + fdmds.add(fdmd); + } + feildrs.close(); + tbmd.setFields(fdmds); + // 扫描主键 + doScanPK(dbName, metadata, tbmd); + // 非空约束 + doScanNN(dbName, metadata, tbmd); + // + doScanUnique(dbName, metadata, tbmd); + // + doScanIndex(dbName, metadata, tbmd); + // + return tbmd; + } + + // 扫描约束，主键；外键；非空；唯一 + @Override + protected Optional doScanPK(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException { + // 主键结果集 + ResultSet pkrs = dbmd.getPrimaryKeys(null, dbName, tbmd.getName()); + // 主键字段 + List pkfs = new ArrayList<>(); + // 主键元数据 + // ResultSetMetaData pkmd = pkrs.getMetaData(); + while(pkrs.next()){ + String pkFn = pkrs.getString(DbSchemaConsts.PK_NAME_INDEX); + Optional op = tbmd.findField(pkFn); + if(op.isPresent()) + pkfs.add(op.get()); + } + // 无主键 + if(pkfs.size()==0) { + return Optional.empty(); + } + tbmd.setPk(new PKConstraintMetadata(tbmd, pkfs)); + return Optional.of(tbmd.getPk()); + } + + @Override + protected Map> doScanFK(String dbName, DatabaseMetaData dbmd, List tbmds) throws SQLException { + Map> r = new HashMap<>(); + // 循环扫描表 + for(TableMetadata tbmd: tbmds) { + List fks = scanFKTable(dbName, dbmd, tbmd, tbmds); + r.put(tbmd.getName(), fks); + } + return r; + } + + private List scanFKTable(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd, List tbmds) throws SQLException { + // 外键元数据结果集 + ResultSet fkrs = dbmd.getImportedKeys(dbName, dbName, tbmd.getName()); + // 主键元数据 + // ResultSetMetaData fkmd = fkrs.getMetaData(); + // + List fkcmds = new ArrayList<>(); + // key: 外键名称 + Map fkcmdMap = new HashMap<>(); + while(fkrs.next()){ + // 外键名称 + String fkName = fkrs.getString(DbSchemaConsts.FK_NAME_INDEX); + // 外键引用表 + String fkTableName = fkrs.getString(DbSchemaConsts.FK_REF_TABLE_INDEX); + TableMetadata refTable = this.findRefTable(fkTableName, tbmds).get(); + // 主表外键字段 + String pfn = fkrs.getString(DbSchemaConsts.FK_PK_FIELD_INDEX); + FieldMetadata pf = tbmd.findField(pfn).get(); + // + String ffn = fkrs.getString(DbSchemaConsts.FK_REF_FIELD_INDEX); + FieldMetadata ff = refTable.findField(ffn).get(); + // + FKField fkf = new FKField(pf, ff); + // 外键存在，即，多字段外键 + if(fkcmdMap.containsKey(fkName)) { + FKConstraintMetadata fkcmd = fkcmdMap.get(fkName); + fkcmd.getFields().add(pf); + fkcmd.getFkFields().add(fkf); + } else { + List fields = new ArrayList<>(); + fields.add(pf); + List fkFields = new ArrayList<>(); + fkFields.add(fkf); + FKConstraintMetadata fkcmd = new FKConstraintMetadata(fkName, tbmd, refTable, fields, fkFields); + fkcmds.add(fkcmd); + fkcmdMap.put(fkName, fkcmd); + } + } + // 外键放入表 + tbmd.setFks(fkcmds); + return fkcmds; + } + + @Override + protected List doScanNN(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException { + // + List nns = new ArrayList<>(); + // 字段元数据 + ResultSet feildrs = dbmd.getColumns(null, dbName, tbmd.getName(), null); + while (feildrs.next()) { + // + String fieldName = feildrs.getString(DbSchemaConsts.FEILD_NAME_INDEX); + int nullable = feildrs.getInt(DbSchemaConsts.FEILD_NULLABLE_INDEX); + // nullable==0 不能为空 + if(nullable==0) { + Optional op = tbmd.findField(fieldName); + if(op.isPresent()) { + nns.add(new NotNullConstraintMetadata(tbmd, op.get())); + } + } + } + feildrs.close(); + tbmd.setNotNull(nns); + return nns; + } + + // 唯一约束 + @Override + protected List doScanUnique(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException { + List uqmds = new ArrayList<>(); + // + ResultSet indexrs = dbmd.getIndexInfo(null, dbName, tbmd.getName(), false, false); + // ResultSetMetaData indexmd = indexrs.getMetaData(); + // + Map uqmdMap = new HashMap<>(); + while (indexrs.next()) { + // 名称，与索引共用 + String unqName = indexrs.getString(DbSchemaConsts.INDEX_NAME_INDEX); + if(unqName==null) + continue; + // nonUnique=true, 即不唯一 + boolean nonUnique = indexrs.getBoolean(DbSchemaConsts.UNIQUE_INDEX); + if (nonUnique) + continue; + String fn = indexrs.getString(DbSchemaConsts.UNIQUE_FIELD_INDEX); + FieldMetadata field = tbmd.findField(fn).get(); + if(uqmdMap.containsKey(unqName)) { + UniqueConstraintMetadata uqmd = uqmdMap.get(unqName); + uqmd.getFields().add(field); + } else { + List fields = new ArrayList<>(); + fields.add(field); + UniqueConstraintMetadata uqmd = new UniqueConstraintMetadata(unqName, tbmd, fields); + uqmds.add(uqmd); + uqmdMap.put(unqName, uqmd); + } + } + tbmd.setUnique(uqmds); + return uqmds; + } + + // 索引 + @Override + protected List doScanIndex(String dbName, DatabaseMetaData dbmd, TableMetadata tbmd) throws SQLException { + List indexmds = new ArrayList<>(); + // + ResultSet indexrs = dbmd.getIndexInfo(null, dbName, tbmd.getName(), false, false); + // + // ResultSetMetaData indexrsmd = indexrs.getMetaData(); + // key: 索引名称 + Map indexmdMap = new HashMap<>(); + while (indexrs.next()) { + // + String indexName = indexrs.getString(DbSchemaConsts.INDEX_NAME_INDEX); + // 非空索引不处理，以非空约束处理 + boolean nonUnique = indexrs.getBoolean(DbSchemaConsts.UNIQUE_INDEX); + if (!nonUnique) + continue; + // 索引字段 + String fn = indexrs.getString(DbSchemaConsts.INDEX_FIELD_INDEX); + Optional o = tbmd.findField(fn); + if(!o.isPresent()) + continue; + // 已有索引存储，多字段索引 + if(indexmdMap.containsKey(indexName)) { + IndexMetadata indexmd = indexmdMap.get(indexName); + indexmd.getFields().add(tbmd.findField(fn).get()); + } else { + List fields = new ArrayList<>(); + fields.add(tbmd.findField(fn).get()); + // 索引类型 + int indexType = indexrs.getInt(DbSchemaConsts.INDEX_TYPE_INDEX); + IndexMetadata indexmd = new IndexMetadata(indexName, tbmd, fields, IndexType.values()[indexType]); + indexmds.add(indexmd); + indexmdMap.put(indexName, indexmd); + } + } + tbmd.setIndexes(indexmds); + return indexmds; + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/SchemaRegistry.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/SchemaRegistry.java new file mode 100644 index 00000000..983f2d5f --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/SchemaRegistry.java @@ -0,0 +1,23 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; + +/** + * @类名: SchemaRegistry + * @说明: schema注册, 支持文件(序列化)，数据库，zk，nacos, redis等等 + * + * @author leehom + * @Date 2022年4月27日 下午3:57:27 + * 修改记录： + * + * @see + */ +public interface SchemaRegistry { + + public DbSchema registry(DbSchema schema); + public DbSchema load(String uri); +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/SchemaUtils.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/SchemaUtils.java new file mode 100644 index 00000000..cd79f1b3 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/SchemaUtils.java @@ -0,0 +1,66 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner; + +import java.util.List; + +import com.leehom.arch.datax.plugin.rdb2graph.common.StringUtils; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; + +public class SchemaUtils { + + /** 字段分割*/ + public static final String FIELD_SEQ = ","; + + // + public static String[] extractTableNames(String prefix, List tables) { + if (tables == null || tables.size() == 0) + return new String[0]; + String[] tns = new String[tables.size()]; + for (int i = 0; i < tables.size(); i++) { + tns[i] = tables.get(i).getName(); + } + return tns; + } + + // 字段名排列 + public static String extractFieldNames(List fields) { + return extractFieldNames(null, fields); + } + + public static String extractFieldNames(String prefix, List fields) { + return extractFieldNames(prefix, fields, FIELD_SEQ); + } + + public static String extractFieldNames(String prefix, List fields, String seq) { + if (fields == null || fields.size() == 0) + return ""; + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < fields.size(); i++) { + String tmp = StringUtils.isNotEmpty(prefix) ? prefix+"."+fields.get(i).getName() : fields.get(i).getName() + seq; + sb.append(tmp); + } + // 去掉最后seq + sb.delete(sb.length()-seq.length(), sb.length()); + return sb.toString(); + } + + // + public static String extractFieldWhereNotNull(List fields, String seq) { + return extractFieldWhereNotNull("", fields, seq); + + } + public static String extractFieldWhereNotNull(String prefix, List fields, String seq) { + if (fields == null || fields.size() == 0) + return ""; + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < fields.size(); i++) { + String fieldItem = fields.get(i).getName() + " is not null "; + String tmp = StringUtils.isNotEmpty(prefix) ? prefix+"."+fieldItem : fieldItem + seq; + sb.append(tmp); + } + // 去掉最后seq + sb.delete(sb.length()-seq.length(), sb.length()); + return sb.toString(); + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/Neo4jDaoConfiguration.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/Neo4jDaoConfiguration.java new file mode 100644 index 00000000..81f499f7 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/Neo4jDaoConfiguration.java @@ -0,0 +1,24 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.config; + +import org.neo4j.driver.AuthTokens; +import org.neo4j.driver.Driver; +import org.neo4j.driver.GraphDatabase; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jDao; + +public class Neo4jDaoConfiguration { + + public static Driver neo4jDriver(final Neo4jDriverProperties driverProperties) { + return GraphDatabase.driver(driverProperties.getUri(), + AuthTokens.basic(driverProperties.getUsername(),driverProperties.getPassword())); + } + + public static Neo4jDao neo4jDao(final Neo4jDriverProperties driverProperties) { + Driver driver = Neo4jDaoConfiguration.neo4jDriver(driverProperties); + Neo4jDao neo4jDao = new Neo4jDao(); + neo4jDao.setDatabase(driverProperties.getDatabase()); + neo4jDao.setDriver(driver); + return neo4jDao; + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/Neo4jDriverProperties.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/Neo4jDriverProperties.java new file mode 100644 index 00000000..f065097b --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/Neo4jDriverProperties.java @@ -0,0 +1,24 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.config; + +import java.net.URI; + +import lombok.Data; + +@Data +public class Neo4jDriverProperties { + + /** + * The uri this driver should connect to. The driver supports bolt or neo4j as schemes. + */ + private URI uri; + private String database; + /** + * The login of the user connecting to the database. + */ + private String username; + + /** + * The password of the user connecting to the database. + */ + private String password; +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/ScannerConfiguration.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/ScannerConfiguration.java new file mode 100644 index 00000000..e09ba6f8 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/ScannerConfiguration.java @@ -0,0 +1,17 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.config; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.AbstractDbScanner; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.MysqlScanner; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.OracleScanner; + +class ScannerConfiguration { + + AbstractDbScanner oracleScanner() { + return new OracleScanner(); + } + + AbstractDbScanner mysqlScanner() { + return new MysqlScanner(); + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/ScannerSerializerConfig.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/ScannerSerializerConfig.java new file mode 100644 index 00000000..7e856049 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/config/ScannerSerializerConfig.java @@ -0,0 +1,53 @@ +/** + * + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.config; + +import java.util.HashMap; +import java.util.Map; + +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.Serializer; +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.xstream.XmlSerializerXStreamImpl; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.NotNullConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.UniqueConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.IndexMetadata; + +/** + * @类名: ScannerSerializerConfig + * @说明: 序列化器配置 + * + * @author leehom + * @Date 2019年10月8日 下午4:21:34 + * 修改记录： + * + * @see + */ +public class ScannerSerializerConfig { + + public static Serializer rdbSchemaXmlSerializer() { + XmlSerializerXStreamImpl ser = new XmlSerializerXStreamImpl(); + // typeAlias + Map typeAlias = new HashMap(); + // schema + typeAlias.put("schema", DbSchema.class); + // TableMetadata + typeAlias.put("table", TableMetadata.class); + // FieldMetadata + typeAlias.put("field", FieldMetadata.class); + // IndexMetadata + typeAlias.put("index", IndexMetadata.class); + // UniqueConstraintMetadata + typeAlias.put("uq", UniqueConstraintMetadata.class); + // FKConstraintMetadata + typeAlias.put("fk", FKConstraintMetadata.class); + // NotNullConstraintMetadata + typeAlias.put("nn", NotNullConstraintMetadata.class); + ser.setTypeAlias(typeAlias); + return ser; + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/graph/Callback.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/graph/Callback.java new file mode 100644 index 00000000..f68602de --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/graph/Callback.java @@ -0,0 +1,23 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.graph; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.graph.Graph.Edge; + +/** + * @类名: Callback + * @说明: 遍历回调 + * + * @author leehom + * @Date 2022年4月29日 下午5:28:00 + * 修改记录： + * + * @see + */ +public interface Callback { + + public R execute(Edge e); + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/graph/Graph.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/graph/Graph.java new file mode 100644 index 00000000..477409c3 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/graph/Graph.java @@ -0,0 +1,111 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.graph; + +import lombok.Data; + +/** + * @类名: Graph + * @说明: 图，邻接表 + * + * @author leehom + * @Date 2022年4月18日 上午10:32:17 + * 修改记录： + * + * @see + */ +public class Graph { + + @SuppressWarnings("rawtypes") + Vertex[] vexs; // 顶点数组 + + /* 顶点*/ + @Data + public static class Vertex { + + public Vertex() { + + } + public Vertex(T data) { + this.data = data; + } + @SuppressWarnings("rawtypes") + Edge first;// 该点指向的第一条边 + boolean selfRef; // 自关联 + T data; // 节点信息 + } + + /* 边*/ + @Data + public static class Edge { + + public Edge(int adjVex) { + this.adjVex = adjVex; + } + public Edge(int adjVex, V data) { + this.adjVex = adjVex; + this.data = data; + } + + int adjVex; // 指向的顶点 + Edge next; // 指向下一个边节点 + V data; + } + + // 增加顶点连接 + public void link(Vertex v, Edge e) { + if(v.first==null) { + v.first = e; + return; + } + Edge l = v.first; + while (l.next != null) { + l = l.next; + } + l.next = e; + } + + /* + * 创建图 vexs 顶点；vrs 边 + */ + public static Graph buildGraph(Vertex[] vexs, VertexRelationship[] vrs) { + Graph g = new Graph(); + g.vexs = vexs; + for(VertexRelationship vr : vrs) { + Edge e = new Edge(vr.getTo(), vr.getData()); + g.link(vexs[vr.getFrom()], e); + } + return g; + } + + // 遍历，集成guava event + public void traversal(Callback callback) { + for (int i = 0; i < vexs.length; i++) { + // 顶点 + Edge e = vexs[i].first; + while (e != null) { + // + callback.execute(e); + e = e.next; + } + } + } + + // 打印 + public void print() { + System.out.printf("Graph:\n"); + for (int i = 0; i < vexs.length; i++) { + // 顶点 + System.out.printf("%d(%s) ", i, vexs[i].data); + Edge e = vexs[i].first; + while (e != null) { + System.out.printf("->%d(%s)", e.adjVex, vexs[e.adjVex].data); + e = e.next; + } + System.out.printf("\n"); + } + } + +} \ No newline at end of file diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/graph/VertexRelationship.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/graph/VertexRelationship.java new file mode 100644 index 00000000..e9d55a0b --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/graph/VertexRelationship.java @@ -0,0 +1,29 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.graph; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * @类名: VertexRelationship + * @说明: 顶点关系 + * + * + * @author leehom + * @Date 2022年1月7日 下午1:35:00 + * 修改记录： + * + * @see + */ +@Data +@AllArgsConstructor +public class VertexRelationship { + + public VertexRelationship(int from, int to) { + this.from = from; + this.to = to; + } + private int from; + private int to; + private V data; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/Neo4jSchema.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/Neo4jSchema.java new file mode 100644 index 00000000..91800baf --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/Neo4jSchema.java @@ -0,0 +1,28 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j; + +import java.util.List; + +import lombok.Data; + +/** + * @类名: Neo4jSchema + * @说明: neo4j图库模式 + * + * + * @author leehom + * @Date 2022年1月7日 下午1:35:00 + * 修改记录： + * + * @see + */ +@Data +public class Neo4jSchema { + + /** 图库名称*/ + private String name; + /** 图库描述*/ + private String remark; + /** 图结构，邻接表*/ + private List nodes; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/NodeMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/NodeMetadata.java new file mode 100644 index 00000000..9b670ebb --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/NodeMetadata.java @@ -0,0 +1,25 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j; + +import java.util.List; + +import lombok.Data; + +/** + * @类名: NodeMetadata + * @说明: 节点元数据 + * + * + * @author leehom + * @Date 2022年1月7日 下午1:35:00 + * 修改记录： + * + * @see + */ +@Data +public class NodeMetadata { + + private String label; + private String remark; + private List relationships; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/PropertyMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/PropertyMetadata.java new file mode 100644 index 00000000..5073cd0c --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/PropertyMetadata.java @@ -0,0 +1,25 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.type.Neo4jType; + +import lombok.Data; + +/** + * @类名: PropertyMetadata + * @说明: 属性元数据 + * + * + * @author leehom + * @Date 2022年1月7日 下午1:35:00 + * 修改记录： + * + * @see + */ +@Data +public class PropertyMetadata { + + private String name; + private Neo4jType type; + private String remark; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/RelationshipMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/RelationshipMetadata.java new file mode 100644 index 00000000..a95b402c --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/RelationshipMetadata.java @@ -0,0 +1,20 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j; + +import lombok.Data; + +/** + * @类名: RelationshipMetadata + * @说明: 关系元数据 + * + * + * @author leehom + * @Date 2022年1月7日 下午1:35:00 + * 修改记录： + * + * @see + */ +@Data +public class RelationshipMetadata { + + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/Neo4jDao.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/Neo4jDao.java new file mode 100644 index 00000000..69c623ad --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/Neo4jDao.java @@ -0,0 +1,137 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds; + +import java.util.List; + +import org.neo4j.driver.AccessMode; +import org.neo4j.driver.Driver; +import org.neo4j.driver.Query; +import org.neo4j.driver.Record; +import org.neo4j.driver.Session; +import org.neo4j.driver.SessionConfig; +import org.neo4j.driver.SessionConfig.Builder; +import org.neo4j.driver.Transaction; +import org.neo4j.driver.exceptions.Neo4jException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.leehom.arch.datax.plugin.rdb2graph.common.StringUtils; + +import lombok.Getter; +import lombok.Setter; + +/** + * @类名: Neo4jDao + * @说明: neo4j dao + * + * @author leehom + * @Date 2022年4月22日 上午11:13:36 + * 修改记录： + * + * @see + * + * TODO + * 1. 返回支持反射转换为bean + */ +public class Neo4jDao { + + private final static Logger log = LoggerFactory.getLogger(Neo4jDao.class); + + @Setter + @Getter + private String database; + + @Setter + private Driver driver; + + // 查询，返回结果集 + public List runQuery(Query query) { + + Builder builder = SessionConfig.builder().withDefaultAccessMode(AccessMode.READ); + SessionConfig sessionConfig; + if(StringUtils.isEmpty(database)) + sessionConfig = builder.build(); + else + sessionConfig = builder.withDatabase(database).build(); + try (Session session = driver.session(sessionConfig)) { + return session.run(query).list(); + } + + } + + // 查询，返回一个结果 + public Record runQuerySingle(Query query) { + + SessionConfig sessionConfig = SessionConfig.builder().withDefaultAccessMode(AccessMode.READ).withDatabase(database).build(); + try (Session session = driver.session(sessionConfig)) { + return session.run(query).single(); + } + + } + + // 执行 + public void executeQuery(Query query) { + + SessionConfig sessionConfig = sessionConfig(AccessMode.WRITE); + try (Session session = driver.session(sessionConfig)) { + session.run(query); + } + + } + + // 事务 + public void runInTransaction(List queries) { + + SessionConfig sessionConfig = SessionConfig.builder().withDefaultAccessMode(AccessMode.WRITE).withDatabase(database).build(); + try (Session session = driver.session(sessionConfig)) { + try(Transaction t = session.beginTransaction()) { + for(Query q : queries) { + t.run(q); + } + t.commit(); + } + } + + } + + public void reTryRunInTransaction(List queries, int retries) { + int r = 0; + while(true) { + try { + SessionConfig sessionConfig = SessionConfig.builder().withDefaultAccessMode(AccessMode.WRITE).withDatabase(database).build(); + try (Session session = driver.session(sessionConfig)) { + try(Transaction t = session.beginTransaction()) { + for(Query q : queries) { + t.run(q); + } + t.commit(); + } + } + break; + } catch (Neo4jException e) { + r++; + if(r>retries) { + log.error("neo4j批量query异常，重试次数超限[{}]，{}", r, e.getMessage()); + break; + } + log.error("neo4j批量query异常[{}]，{}", r, e.getMessage()); + try { + Thread.sleep(300); + } catch (InterruptedException e1) { + + } + } + } // end while + + } + + private SessionConfig sessionConfig(AccessMode mode) { + Builder builder = SessionConfig.builder().withDefaultAccessMode(mode); + SessionConfig sessionConfig; + if(StringUtils.isEmpty(database)) + sessionConfig = builder.build(); + else + sessionConfig = builder.withDatabase(database).build(); + return sessionConfig; + } +} + diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/Neo4jQueryPattern.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/Neo4jQueryPattern.java new file mode 100644 index 00000000..ed9eca3e --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/Neo4jQueryPattern.java @@ -0,0 +1,58 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds; + + +/** + * @类名: Neo4jQueryPattern + * @说明: neo4j查询 + * + * @author leehom + * @Date 2013-4-10 下午2:31:41 + * @修改记录： + * + * @see + */ +public interface Neo4jQueryPattern { + + /** 数据库*/ + public static final String USE_DB = "USE $dbname"; + public static final String CREATE_DB = "CREATE DATABASE $dbname IF NOT EXISTS"; + + + /** node属性别名*/ + public static final String PROPERTTY_ALIAS = "n."; + /** 关系属性别名*/ + public static final String REL_PROPERTTY_ALIAS = "r."; + + /** + * 创建唯一约束 + * constraintname;labelname;properties + */ + public static final String CREATE_UNQ_CONSTRAINT = "CREATE CONSTRAINT {0} IF NOT EXISTS " + + "FOR (n:{1}) " + + "REQUIRE ({2}) IS UNIQUE"; + + /** 创建非空约束，constraintname;labelname;properties*/ + public static final String CREATE_NN_CONSTRAINT = "CREATE CONSTRAINT {0} IF NOT EXISTS " + + "FOR (n:{1}) " + + "REQUIRE ({2}) IS NOT NULL"; + + // node 索引，0, 索引名称；1, n:Lable; 2, n.x, 节点属性 + public static final String CREATE_NODE_INDEX = "CREATE INDEX {0} IF NOT EXISTS FOR (n:{1}) ON ({2})"; + // 关系 索引， 0, 索引名称；1, n:Lable; 2, n.x, 关系属性 + public static final String CREATE_REL_INDEX = "CREATE INDEX {0} IF NOT EXISTS FOR ()-[n:{1}]-() ON ({2})"; + + // node, + // 0, label; 1, 属性, {"key", "value"} + public static final String CREATE_NODE = "CREATE (n:{0} {1})"; + public static final String UPDATE_NODE = "CREATE (n:{0} {1})"; + public static final String DELETE_NODE = "MATCH (n:{0} {1}) DELETE n"; + + // rel，0，关系from node label；1，关系to node label；2， where， from/to；3, 关系type；4， 关系属性 + public static final String CREATE_REL = "MATCH (a:{0}), (b:{1}) WHERE {2} " + + "CREATE (a)-[r:{3} {4}]->(b) "; + + // query all + public static final String MATCH_ALL = "MATCH (n) RETURN n"; + + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/ParamsUtils.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/ParamsUtils.java new file mode 100644 index 00000000..6d54afbb --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/ParamsUtils.java @@ -0,0 +1,63 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; +import java.util.Map; + +import com.leehom.arch.datax.plugin.rdb2graph.common.DateTimeUtils; + +/** + * @类名: ParamsUtils + * @说明: 参数工具类 + * + * @author leehom + * @Date 2022年5月5日 下午3:48:14 + * 修改记录： + * + * @see + */ +public class ParamsUtils { + + // 日期类型转换 + public static LocalDateTime dateToLocalDateTime(Date date) { + if(date==null) + return null; + Instant instant = date.toInstant(); + ZoneId zoneId = ZoneId.systemDefault(); + return instant.atZone(zoneId).toLocalDateTime(); + } + + /** */ + public static final String FIELD_SEQ = ","; + /** */ + public static final String FIELD_QUOTA = "'"; + + // 参数字符串 + public static String params2String(Map params) { + if (params == null || params.size() == 0) + return ""; + StringBuffer sb = new StringBuffer("{"); + for(Map.Entry entry : params.entrySet()){ + String name = entry.getKey(); + Object v = entry.getValue(); + if(v==null) + continue; + // 数字，int；long；double，不需要引号 + if(Number.class.isAssignableFrom(v.getClass())) { + sb.append(name).append(":").append(v.toString()); + } else if(v instanceof Date) { + String dataStr = DateTimeUtils.DateToString((Date)v, DateTimeUtils.ISO8601PattenNoZone); + sb.append(name).append(":").append("'").append(dataStr).append("'"); + } else { // + sb.append(name).append(":").append("'").append(v.toString()).append("'"); + } + sb.append(FIELD_SEQ); + } + sb.delete(sb.length()-1, sb.length()); + sb.append("}"); + return sb.toString(); + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/QueryType.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/QueryType.java new file mode 100644 index 00000000..5d69ab35 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/QueryType.java @@ -0,0 +1,21 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds; + +/** + * @类名: TypeEnum + * @说明: 参数类型枚举 + * + * @author leehom + * @Date 2020年9月3日 下午6:01:12s + * 修改记录： + * + * @see + */ +public enum QueryType { + + QUERY, SINGLE_QUERY, INSERT, UPDATE, DELETE; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/QueryWrapper.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/QueryWrapper.java new file mode 100644 index 00000000..7f679f51 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/ds/QueryWrapper.java @@ -0,0 +1,32 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds; + +import org.neo4j.driver.Query; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * @类名: QueryWrapper + * @说明: neo4j query包装 + * + * @author leehom + * @Date 2022年4月22日 下午6:07:23 + * 修改记录： + * + * @see + */ +@Data +@AllArgsConstructor +public class QueryWrapper { + + private Query query; + private QueryType queryType; + + public static QueryWrapper wrap(Query query, QueryType queryType) { + return new QueryWrapper(query, queryType); + } +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/type/JdbcTypeCast.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/type/JdbcTypeCast.java new file mode 100644 index 00000000..5d1af50b --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/type/JdbcTypeCast.java @@ -0,0 +1,24 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.type; + +import java.sql.JDBCType; + +/** + * @类名: JdbcTypeCast + * @说明: jdbc类型转换为neo4j类型 + * + * @author leehom + * @Date 2020年9月3日 下午3:42:26 + * 修改记录： + * + * @see + */ +@FunctionalInterface +public interface JdbcTypeCast { + + public Neo4jType cast(JDBCType jdbcType); + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/type/JdbcTypeCastRegistry.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/type/JdbcTypeCastRegistry.java new file mode 100644 index 00000000..234bcfb2 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/type/JdbcTypeCastRegistry.java @@ -0,0 +1,19 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.type; + +/** + * @类名: JdbcTypeCastRegistry + * @说明: jdbc类型转换注册器 + * + * @author leehom + * @Date 2020年9月3日 下午3:42:26 + * 修改记录： + * + * @see + */ +public class JdbcTypeCastRegistry { + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/type/Neo4jType.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/type/Neo4jType.java new file mode 100644 index 00000000..d81d7216 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/neo4j/type/Neo4jType.java @@ -0,0 +1,78 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.type; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * @类名: TypeEnum + * @说明: 参数类型枚举 + * + * @author leehom + * @Date 2020年9月3日 下午6:01:12s + * 修改记录： + * + * @see + */ +/** + * neo4j类型 v4.0 + * https://neo4j.com/docs/cypher-manual/4.0/syntax/values/ + */ +public enum Neo4jType { + + BOOLEAN("boolean", "布尔型", Boolean.class), + SHORT("short", "短整型", Short.class), + INT("int", "整型", Integer.class), + LONG("long", "长整型", Long.class), + FLOAT("float", "浮点型", Float.class), + DOUBLE("double", "双精度浮点型", Double.class), + STRING("string", "字符串型", String.class), + DATE("date", "日期型", Date.class), + ENUMCONST("enumconst", "枚举项", Enum.class), + ENUM("enum", "枚举", Enum.class), + // composite type + STRINGS("strings", "字符串型数组", String[].class); + + private static Map> nameClazzMap; + private static Map typeMap; + + static { + nameClazzMap = new HashMap<>(); + nameClazzMap.put("BOOLEAN", Boolean.class); + nameClazzMap.put("SHORT", Short.class); + nameClazzMap.put("INT", Integer.class); + nameClazzMap.put("LONG", Long.class); + nameClazzMap.put("FLOAT", Float.class); + nameClazzMap.put("DOUBLE", Double.class); + nameClazzMap.put("STRING", String.class); + nameClazzMap.put("DATE", Date.class); + // + typeMap = new HashMap<>(); + typeMap.put("BOOLEAN", BOOLEAN); + typeMap.put("SHORT", SHORT); + typeMap.put("INT", INT); + typeMap.put("LONG", LONG); + typeMap.put("FLOAT", FLOAT); + typeMap.put("DOUBLE", DOUBLE); + + } + + // 获取类型 + public static Class nameToClazz(String name) { + return nameClazzMap.get(name); + } + + // + public static Neo4jType nameToType(String name) { + return typeMap.get(name); + } + + Neo4jType(String name, String alias, Class clazz) { + + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/DbSchema.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/DbSchema.java new file mode 100644 index 00000000..81c962fa --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/DbSchema.java @@ -0,0 +1,106 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb; + +import java.util.ArrayList; +import java.util.List; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.graph.Graph; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.graph.VertexRelationship; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.graph.Graph.Vertex; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKField; + +import lombok.Data; + +/** + * @类名: DbSchema + * @说明: 数据库模式 + * + * + * @author leehom + * @Date 2022年1月7日 下午1:35:00 + * 修改记录： + * + * @see + */ +@Data +public class DbSchema { + + /** */ + private String name; + /** 注释*/ + private String remark; + /** 数据库表*/ + private List tables; + + /** + * @说明：设置/去除表为连接表 + * + * @author leehom + * @param table + * @param linkFromFK + * + */ + public void setLinkTable(TableMetadata table, FKConstraintMetadata linkFromFK) { + // 验证是否符合连接表， node->edge->edge + // TODO 统一异常规范 + if(table.getFks()==null||!(table.getFks().size()==2)) { + throw new RuntimeException("不符合连接表形态，node->edge->edge"); + } + table.setLinkTable(true); + table.setLinkFrom(linkFromFK); + } + + public void unSetLinkTable(TableMetadata table) { + table.setLinkTable(false); + table.setLinkFrom(null); + } + + // 计算表连接图 + @SuppressWarnings("unchecked") + public Graph buildTableGraph() { + Vertex[] vxs = new Vertex[tables.size()]; + // 构建顶点 + for(int i=0;i(tables.get(i).getName()); + } + // 构建关系 + List>> vrs = new ArrayList<>(); + for(int i=0;i fks = table.getFks(); + int from = i; + for(FKConstraintMetadata fk : fks) { + // + int to = locate(fk.getRefTable().getName(), vxs); + // 构建关系 + VertexRelationship> vr = new VertexRelationship>(from, to, fk.getFkFields()); + vrs.add(vr); + } + + } + // 构建表连接图 + return Graph.buildGraph(vxs, vrs.toArray(new VertexRelationship[0])); + } + + // 获取表邻接数组位置 + private int locate(String tableName, Vertex[] vxs) { + int l = 0; + for(Vertex vx : vxs) { + if(tableName.equals(vx.getData())) + break; + l++; + } + return l; + + } + + public TableMetadata findTable(String name) { + for(TableMetadata t : tables) { + if(t.getName().equals(name)) { + return t; + } + } + return null; + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/DbSchemaConsts.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/DbSchemaConsts.java new file mode 100644 index 00000000..1a6b9c3e --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/DbSchemaConsts.java @@ -0,0 +1,60 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb; + + +/** + * @类名: DbSchemaConsts + * @说明: 数据库元数据常量 + * + * @author leehom + * @Date 2013-4-10 下午2:31:41 + * @修改记录： + * + * @see + */ +public interface DbSchemaConsts { + + /** */ + public static final int DB_NAME_INDEX = 1; + public static final int DB_COMMENTS_INDEX = 9; + + public static final int TABLE_NAME_INDEX = 3; + public static final int TABLE_COMMENTS_INDEX = 5; + // 字段 + public static final int FEILD_NAME_INDEX = 4; + public static final int FEILD_DATA_TYPE_INDEX = 5; + public static final int FEILD_LENGTH_INDEX = 7; + public static final int FEILD_NULLABLE_INDEX = 11; + public static final int FEILD_REMARK_INDEX = 12; + + /** 主键约束*/ + public static final int PK_NAME_INDEX = 4; + + /** 外键约束*/ + /** */ + public static final int FK_FIELD_COUNT = 14; + /** 主表键名称*/ + public static final int FK_PK_FIELD_INDEX = 8; + /** 外键引用表名称*/ + public static final int FK_REF_TABLE_INDEX = 3; + /** 外键引用字段名称*/ + public static final int FK_REF_FIELD_INDEX = 4; + /** 外键名称*/ + public static final int FK_NAME_INDEX = 12; + + // 唯一约束 + /** 元数据结果集字段数*/ + public static final int UNIQUE_FIELD_COUNT = 14; + public static final int UNIQUE_INDEX = 4; + public static final int UNIQUE_FIELD_INDEX = 9; + + /** 索引*/ + public static final int INDEX_FIELD_COUNT = 14; + public static final int INDEX_NAME_INDEX = 6; + public static final int INDEX_FIELD_INDEX = 9; + public static final int INDEX_TYPE_INDEX = 7; + + // 表类型， + public static final String TABLE_TYPE = "TABLE"; + + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/FieldMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/FieldMetadata.java new file mode 100644 index 00000000..47034b19 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/FieldMetadata.java @@ -0,0 +1,29 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb; + +import java.sql.JDBCType; + +import lombok.Data; + +/** + * @类名: FieldMetadata + * @说明: 字段元数据 + * + * + * @author leehom + * @Date 2022年1月7日 下午1:35:00 + * 修改记录： + * + * @see + */ +@Data +public class FieldMetadata { + + private String name; + /** 类型*/ + private JDBCType type; + /** 长度*/ + private Integer length; + /** 注释*/ + private String remark; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/MultiFieldsTableComponent.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/MultiFieldsTableComponent.java new file mode 100644 index 00000000..73b4e4fe --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/MultiFieldsTableComponent.java @@ -0,0 +1,35 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb; + +import java.util.List; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @类名: MultiFieldsDbSchemaComponent + * @说明: 多字段数据库构件 + * + * + * @author leehom + * @Date 2022年4月16日 下午6:03:20 + * 修改记录： + * + * @see + */ +@Data +@EqualsAndHashCode(callSuper=false) +public class MultiFieldsTableComponent extends TableComponent { + + public MultiFieldsTableComponent(TableMetadata tbmd, List fields) { + super(tbmd); + this.fields = fields; + } + + /** 约束作用字段*/ + private List fields; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/SingleFieldTableComponent.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/SingleFieldTableComponent.java new file mode 100644 index 00000000..caacbd1b --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/SingleFieldTableComponent.java @@ -0,0 +1,33 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @类名: SingleFieldDbSchemaComponent + * @说明: 单字段数据库构件 + * + * + * @author leehom + * @Date 2022年4月16日 下午6:03:20 + * 修改记录： + * + * @see + */ +@Data +@EqualsAndHashCode(callSuper=false) +public class SingleFieldTableComponent extends TableComponent { + + public SingleFieldTableComponent(TableMetadata tbmd, FieldMetadata field) { + super(tbmd); + this.field = field; + } + + /** 约束作用字段*/ + private FieldMetadata field; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/TableComponent.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/TableComponent.java new file mode 100644 index 00000000..11c0f701 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/TableComponent.java @@ -0,0 +1,29 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * @类名: TableComponent + * @说明: 表功能构件 + * 约束，保证数据完整性 + * 索引，加快数据搜索 + * + * + * @author leehom + * @Date 2022年4月16日 下午6:03:20 + * 修改记录： + * + * @see + */ +@Data +@AllArgsConstructor +public abstract class TableComponent { + + /** 所属表*/ + private TableMetadata tbmd; +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/TableMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/TableMetadata.java new file mode 100644 index 00000000..63f4d1b7 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/TableMetadata.java @@ -0,0 +1,70 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb; + +import java.util.List; +import java.util.Optional; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.NotNullConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.PKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.UniqueConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.IndexMetadata; + +import lombok.Data; +import lombok.Getter; + +/** + * @类名: TableMetadata + * @说明: 数据库表元数据 + * + * + * @author leehom + * @Date 2022年1月7日 下午1:35:00 + * 修改记录： + * + * @see + */ +@Data +public class TableMetadata { + + /** 表名称*/ + private String name; + /** 注释*/ + private String remark; + /** 字段元数据*/ + private List fields; + // 约束 + /** 主键*/ + private PKConstraintMetadata pk; + /** 外键*/ + private List fks; + /** 非空*/ + private List notNull; + /** 唯一*/ + private List unique; + // 索引 + private List indexes; + /** + * 是否连接表 + * link->t1->t2 + */ + private boolean isLinkTable; + /** 连接表有效，连接关系起点外键*/ + private FKConstraintMetadata linkFrom; + + public Optional findField(String name) { + for(FieldMetadata fmd : fields) { + if(fmd.getName().equals(name)) + return Optional.of(fmd); + } + return Optional.empty(); + } + + public FKConstraintMetadata findFk(String name) { + for(FKConstraintMetadata fkmd : fks) { + if(fkmd.getFkName().equals(name)) + return fkmd; + } + return null; + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/NotNullConstraintMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/NotNullConstraintMetadata.java new file mode 100644 index 00000000..8d0c93f7 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/NotNullConstraintMetadata.java @@ -0,0 +1,37 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.SingleFieldTableComponent; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @类名: NotNullConstraintMetadata + * @说明: 非空约束元数据 + * + * @author leehom + * @Date 2022年4月16日 下午6:03:20 + * 修改记录： + * + * @see + */ +@Data +@EqualsAndHashCode(callSuper=false) +public class NotNullConstraintMetadata extends SingleFieldTableComponent { + + public static final String NN_NAME = "_NN"; + + public NotNullConstraintMetadata(TableMetadata tbmd, FieldMetadata field) { + super(tbmd, field); + this.nnName = field.getName().toUpperCase()+NN_NAME; + } + + private String nnName; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/PKConstraintMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/PKConstraintMetadata.java new file mode 100644 index 00000000..d5ee00df --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/PKConstraintMetadata.java @@ -0,0 +1,33 @@ + /** + * + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint; + +import java.util.List; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.MultiFieldsTableComponent; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @类名: PKConstraintMetadata + * @说明: 主键约束元数据 + * + * @author leehom + * @Date 2022年4月16日 下午6:03:20 + * 修改记录： + * + * @see + */ +@Data +@EqualsAndHashCode(callSuper=false) +public class PKConstraintMetadata extends MultiFieldsTableComponent { + + public PKConstraintMetadata(TableMetadata table, List fields) { + super(table, fields); + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/UniqueConstraintMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/UniqueConstraintMetadata.java new file mode 100644 index 00000000..ed6e142f --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/UniqueConstraintMetadata.java @@ -0,0 +1,37 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint; + +import java.util.List; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.MultiFieldsTableComponent; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @类名: UniqueConstraintMetadata + * @说明: 唯一约束元数据, 多字段 + * + * @author leehom + * @Date 2022年4月16日 下午6:03:20 + * 修改记录： + * + * @see + */ +@Data +@EqualsAndHashCode(callSuper=false) +public class UniqueConstraintMetadata extends MultiFieldsTableComponent { + + public UniqueConstraintMetadata(String unqName, TableMetadata table, List fields) { + super(table, fields); + this.unqName = unqName; + } + + private String unqName; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/fk/FKConstraintMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/fk/FKConstraintMetadata.java new file mode 100644 index 00000000..78948879 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/fk/FKConstraintMetadata.java @@ -0,0 +1,43 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk; + +import java.util.List; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.MultiFieldsTableComponent; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @类名: FKConstraintMetadata + * @说明: 外键约束元数据 + * + * @author leehom + * @Date 2022年4月16日 下午6:03:20 + * 修改记录： + * + * @see + */ +@Data +@EqualsAndHashCode(callSuper=false) +public class FKConstraintMetadata extends MultiFieldsTableComponent { + + public FKConstraintMetadata(String name, TableMetadata table, TableMetadata refTable, List fields, List fkFields) { + super(table, fields); + this.fkName = name; + this.refTable = refTable; + this.fkFields = fkFields; + } + /** 外键名称*/ + private String fkName; + /** */ + private TableMetadata refTable; + /** 字段元数据*/ + private List fkFields; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/fk/FKField.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/fk/FKField.java new file mode 100644 index 00000000..3886823f --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/constraint/fk/FKField.java @@ -0,0 +1,31 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * @类名: FKField + * @说明: 外键字段 + * + * @author leehom + * @Date 2022年4月16日 下午6:03:20 + * 修改记录： + * + * @see + */ +@Data +@AllArgsConstructor +public class FKField { + + /** */ + private FieldMetadata field; + /** 参考字段*/ + private FieldMetadata refField; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/BaseConnectionProperties.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/BaseConnectionProperties.java new file mode 100644 index 00000000..5f720a3a --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/BaseConnectionProperties.java @@ -0,0 +1,36 @@ +/** + * %项目描述% + * %ver% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds; + +import lombok.Data; + +/** + * @类名: DruidConnectionProperties + * @说明: druid连接池属性 + * + * + * @author: leehom + * @Date 2018年10月22日 下午4:39:19 + * @修改记录： + * + * @see + * + * + */ +@Data +public class BaseConnectionProperties { + + private Long id; + private String name; + /** jdbc连接url*/ + private String url; + /** 驱动类类型*/ + private String driverClass; + /** 账号&密码*/ + private String userName; + private String password; + private String remark; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/BaseDataSourceBean.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/BaseDataSourceBean.java new file mode 100644 index 00000000..330b93bb --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/BaseDataSourceBean.java @@ -0,0 +1,29 @@ +/** + * %项目描述% + * %ver% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds; + +import javax.sql.DataSource; + +import lombok.Data; + +/** + * @类名: BasicDataSource + * @说明: 数据源 + * + * + * @author: leehom + * @Date 2018年10月22日 下午4:35:30 + * @修改记录： + * + * @see + * + * + */ +@Data +public abstract class BaseDataSourceBean { + + public abstract DataSource toDataSource() throws Exception; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/DruidConnectionProperties.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/DruidConnectionProperties.java new file mode 100644 index 00000000..208b69d7 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/DruidConnectionProperties.java @@ -0,0 +1,65 @@ +/** + * %项目描述% + * %ver% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds; + +import java.util.List; + +import lombok.Data; + +/** + * @类名: DruidConnectionProperties + * @说明: druid连接池属性 + * + * + * @author: leehom + * @Date 2018年10月22日 下午4:39:19 + * @修改记录： + * + * @see + * + * + */ +@Data +public class DruidConnectionProperties { + private Long id; + private String name; + /** 目标数据库*/ + private String db; + /** 表，白名单*/ + private List tbs; + /** jdbc连接url*/ + private String url; + /** 驱动类类型*/ + private String driverClass; + /** 账号&密码*/ + private String userName; + private String password; + private String remark; + /** 初始连接数*/ + private int initialSize = 1; + /** 最大请求等待时间*/ + private int maxWait = 60000; + /** 最少空闲连接数*/ + private int minIdle = 1; + /** 检查空闲连接频率*/ + private int timeBetweenEvictionRunsMillis = 60000; + /** 最小空闲时间, 10分钟*/ + private int minEvictableIdleTimeMillis = 600000; + /** 最大空闲时间, 60分钟*/ + private int maxEvictableIdleTimeMillis = 3600000; + /** */ + private String validationQuery = "select 'x'"; + /** 有效性测试*/ + private boolean testWhileIdle = true; + /** 有效性测试*/ + private boolean testOnBorrow = false; + /** 有效性测试*/ + private boolean testOnReturn = false; + /** 缓存sql*/ + private boolean poolPreparedStatements = true; + /** 最多缓存SQL数*/ + private int maxOpenPreparedStatements = 20; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/DruidDataSourceBean.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/DruidDataSourceBean.java new file mode 100644 index 00000000..18288946 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/ds/DruidDataSourceBean.java @@ -0,0 +1,59 @@ +/** + * %项目描述% + * %ver% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds; + +import javax.sql.DataSource; + +import com.alibaba.druid.pool.DruidDataSource; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @类名: DataSource + * @说明: 数据源 + * + * + * @author: leehom + * @Date 2018年10月22日 下午4:35:30 + * @修改记录： + * + * @see + * + * + */ +@Data +@EqualsAndHashCode(callSuper=false) +public class DruidDataSourceBean extends BaseDataSourceBean { + + /** 连接属性*/ + private DruidConnectionProperties connProps; + + @Override + public DataSource toDataSource() throws Exception { + DruidDataSource druidDataSource = new DruidDataSource(); + druidDataSource.setDriverClassName(connProps.getDriverClass()); + druidDataSource.setUrl(connProps.getUrl()); + druidDataSource.setUsername(connProps.getUserName()); + druidDataSource.setPassword(connProps.getPassword()); + druidDataSource.setInitialSize(connProps.getInitialSize()); + druidDataSource.setMinIdle(connProps.getMinIdle()); + // druidDataSource.setMaxActive(Integer.parseInt(propertyResolver.getProperty("maxActive"))); + druidDataSource.setMaxWait(connProps.getMaxWait()); + druidDataSource.setTimeBetweenEvictionRunsMillis(connProps.getTimeBetweenEvictionRunsMillis()); + druidDataSource.setMinEvictableIdleTimeMillis(connProps.getMinEvictableIdleTimeMillis()); + druidDataSource.setValidationQuery(connProps.getValidationQuery()); + druidDataSource.setTestWhileIdle(connProps.isTestWhileIdle()); + druidDataSource.setTestOnBorrow(connProps.isTestOnBorrow()); + druidDataSource.setTestOnReturn(connProps.isTestOnReturn()); + druidDataSource.setPoolPreparedStatements(connProps.isPoolPreparedStatements()); + druidDataSource.setMaxPoolPreparedStatementPerConnectionSize(connProps.getMaxOpenPreparedStatements()); + druidDataSource.setKeepAlive(true); + druidDataSource.init(); + return druidDataSource; + + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/IndexMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/IndexMetadata.java new file mode 100644 index 00000000..24e962ff --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/IndexMetadata.java @@ -0,0 +1,39 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index; + +import java.util.List; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.MultiFieldsTableComponent; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @类名: IndexMetadata + * @说明: 索引元数据，多字段 + * + * @author leehom + * @Date 2022年4月16日 下午6:03:20 + * 修改记录： + * + * @see + */ +@Data +@EqualsAndHashCode(callSuper=false) +public class IndexMetadata extends MultiFieldsTableComponent { + + public IndexMetadata(String name, TableMetadata table, List fields, IndexType type) { + super(table, fields); + this.name = name; + this.type = type; + } + + private String name; + private IndexType type; + +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/IndexOrder.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/IndexOrder.java new file mode 100644 index 00000000..eb61cd00 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/IndexOrder.java @@ -0,0 +1,31 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index; + + +/** + * @类名: IndexOrder + * @说明: + * + * @author leehom + * @Date 2022年4月19日 下午12:06:13 + * 修改记录： + * + * @see + */ +public enum IndexOrder { + + ASC("ASC"), DESC("DESC"); + + String value; + + private IndexOrder(String value) { + this.value = value; + } + + public String value() { + return this.value; + } + + public String getValue() { + return this.value; + } +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/IndexType.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/IndexType.java new file mode 100644 index 00000000..2ca47372 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/IndexType.java @@ -0,0 +1,40 @@ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index; + + +/** + * @类名: IndexMode + * @说明: + * + * @author leehom + * @Date 2022年4月19日 下午12:06:13 + * 修改记录： + * + * @see + */ + +//public static final short tableIndexStatistic = 0; +// +//public static final short tableIndexClustered = 1; +// +//public static final short tableIndexHashed = 2; +// +//public static final short tableIndexOther = 3; + +public enum IndexType { + + STATISTIC("statistic"), CLUSTERED("clustered"), HASHED("hashed"), OTHER("other"); + + private String value; + + private IndexType(String value) { + this.value = value; + } + + public String value() { + return this.value; + } + + public String getValue() { + return this.value; + } +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/NodeIndexMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/NodeIndexMetadata.java new file mode 100644 index 00000000..f9d8100a --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/NodeIndexMetadata.java @@ -0,0 +1,25 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * @类名: NodeIndexMetadata + * @说明: + * + * @author leehom + * @Date 2022年4月25日 上午9:57:20 + * 修改记录： + * + * @see + */ +@Data +@AllArgsConstructor +public class NodeIndexMetadata { + + private IndexMetadata indexmd; +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/RelIndexMetadata.java b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/RelIndexMetadata.java new file mode 100644 index 00000000..0558538f --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/rdb/index/RelIndexMetadata.java @@ -0,0 +1,25 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * @类名: RelIndexMetadata + * @说明: + * + * @author leehom + * @Date 2022年4月25日 上午9:57:20 + * 修改记录： + * + * @see + */ +@Data +@AllArgsConstructor +public class RelIndexMetadata { + + private IndexMetadata indexmd; +} diff --git a/rdb2graph/rdb2graph-scanner/src/main/resources/logback.xml b/rdb2graph/rdb2graph-scanner/src/main/resources/logback.xml new file mode 100644 index 00000000..0f648972 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/resources/logback.xml @@ -0,0 +1,76 @@ + + + + spring-boot-log + + + + + + [%date] %clr([%level]) %clr([%logger]:%L) >>> %msg %n + utf-8 + + + + INFO + + + + + logs/info.log + + + + logs/info-%d{yyyy-MM-dd}.%i.log + + 128MB + + + 30 + + true + + [%date] %clr([%level]) %clr([%logger]:%L) >>> %msg %n + utf-8 + + + info + ACCEPT + DENY + + + + + logs/error.log + + + + logs/error-%d{yyyy-MM-dd}.%i.log + + 2MB + + + 180 + + true + + + [%date] %clr([%level]) [%thread] %clr([%logger]:%L) >>> %msg %n + utf-8 + + + + + ERROR + + ACCEPT + + DENY + + + + + + + + diff --git a/rdb2graph/rdb2graph-scanner/src/main/resources/sakila.xml b/rdb2graph/rdb2graph-scanner/src/main/resources/sakila.xml new file mode 100644 index 00000000..65ef1eaa --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/main/resources/sakila.xml @@ -0,0 +1,1849 @@ + + sakila + +
+ actor + + + actor_id + SMALLINT + 5 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + ACTOR_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_actor_last_name + OTHER + + + false +
+ + address + + + address_id + SMALLINT + 5 + + + + address + VARCHAR + 50 + + + + address2 + VARCHAR + 50 + + + + district + VARCHAR + 20 + + + + city_id + SMALLINT + 5 + + + + postal_code + VARCHAR + 10 + + + + phone + VARCHAR + 20 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_address_city + + city + + + city_id + SMALLINT + 5 + + + + city + VARCHAR + 50 + + + + country_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_city_country + + country + + + country_id + SMALLINT + 5 + + + + country + VARCHAR + 50 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + COUNTRY_ID_NN + + + + + COUNTRY_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false + + + + + + + + + + + + + + CITY_ID_NN + + + + + CITY_NN + + + + + COUNTRY_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_country_id + OTHER + + + false + + + + + + + + + + + + + + ADDRESS_ID_NN + + + + + ADDRESS_NN + + + + + DISTRICT_NN + + + + + CITY_ID_NN + + + + + PHONE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_city_id + OTHER + + + false +
+ + category + + + category_id + TINYINT + 3 + + + + name + VARCHAR + 25 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + CATEGORY_ID_NN + + + + + NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false +
+ +
+
+ customer + + + customer_id + SMALLINT + 5 + + + + store_id + TINYINT + 3 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + email + VARCHAR + 50 + + + + address_id + SMALLINT + 5 + + + + active + BIT + 0 + + + + create_date + TIMESTAMP + 19 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_customer_address + + + + + + + + + + + + + + fk_customer_store + + store + + + store_id + TINYINT + 3 + + + + manager_staff_id + TINYINT + 3 + + + + address_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_store_address + + + + + + + + + + + + + + fk_store_staff + + staff + + + staff_id + TINYINT + 3 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + address_id + SMALLINT + 5 + + + + picture + LONGVARBINARY + 65535 + + + + email + VARCHAR + 50 + + + + store_id + TINYINT + 3 + + + + active + BIT + 0 + + + + username + VARCHAR + 16 + + + + password + VARCHAR + 40 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_staff_address + + + + + + + + + + + + + + fk_staff_store + + + + + + + + + + + + + + STAFF_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + ADDRESS_ID_NN + + + + + STORE_ID_NN + + + + + ACTIVE_NN + + + + + USERNAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + + + + + idx_fk_store_id + OTHER + + + false + + + + + + + + + + + + + + STORE_ID_NN + + + + + MANAGER_STAFF_ID_NN + + + + + ADDRESS_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + idx_unique_manager + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + false + + + + + + + + + + + + + + CUSTOMER_ID_NN + + + + + STORE_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + ADDRESS_ID_NN + + + + + ACTIVE_NN + + + + + CREATE_DATE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + + + + + idx_fk_store_id + OTHER + + + + + + + idx_last_name + OTHER + + + false +
+ + film + + + film_id + SMALLINT + 5 + + + + title + VARCHAR + 128 + + + + description + LONGVARCHAR + 65535 + + + + release_year + DATE + 0 + + + + language_id + TINYINT + 3 + + + + original_language_id + TINYINT + 3 + + + + rental_duration + TINYINT + 3 + + + + rental_rate + DECIMAL + 4 + + + + length + SMALLINT + 5 + + + + replacement_cost + DECIMAL + 5 + + + + rating + CHAR + 5 + + + + special_features + CHAR + 54 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_film_language + + language + + + language_id + TINYINT + 3 + + + + name + CHAR + 20 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + LANGUAGE_ID_NN + + + + + NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false + + + + + + + + + + + + + + fk_film_language_original + + + + + + + + + + + + + + FILM_ID_NN + + + + + TITLE_NN + + + + + LANGUAGE_ID_NN + + + + + RENTAL_DURATION_NN + + + + + RENTAL_RATE_NN + + + + + REPLACEMENT_COST_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_language_id + OTHER + + + + + + + idx_fk_original_language_id + OTHER + + + + + + + idx_title + OTHER + + + false +
+ + film_actor + + + actor_id + SMALLINT + 5 + + + + film_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + + fk_film_actor_actor + + + + + + + + + + + + + + fk_film_actor_film + + + + + + + + + + + + + + ACTOR_ID_NN + + + + + FILM_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_film_id + OTHER + + + true + +
+ + film_category + + + film_id + SMALLINT + 5 + + + + category_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + + fk_film_category_category + + + + + + + + + + + + + + fk_film_category_film + + + + + + + + + + + + + + FILM_ID_NN + + + + + CATEGORY_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + + PRIMARY + + + + + + + + + fk_film_category_category + OTHER + + + true + +
+ + film_text + + + film_id + SMALLINT + 5 + + + + title + VARCHAR + 255 + + + + description + LONGVARCHAR + 65535 + + + + + + + + + + + + + + + FILM_ID_NN + + + + + TITLE_NN + + + + + + + + + PRIMARY + + + + + + + + + + idx_title_description + OTHER + + + false +
+ + inventory + + + inventory_id + INTEGER + 8 + + + + film_id + SMALLINT + 5 + + + + store_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_inventory_film + + + + + + + + + + + + + + fk_inventory_store + + + + + + + + + + + + + + INVENTORY_ID_NN + + + + + FILM_ID_NN + + + + + STORE_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_film_id + OTHER + + + + + + + + idx_store_id_film_id + OTHER + + + false +
+ +
+ payment + + + payment_id + SMALLINT + 5 + + + + customer_id + SMALLINT + 5 + + + + staff_id + TINYINT + 3 + + + + rental_id + INTEGER + 10 + + + + amount + DECIMAL + 5 + + + + payment_date + TIMESTAMP + 19 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_payment_customer + + + + + + + + + + + + + + fk_payment_rental + + rental + + + rental_id + INTEGER + 10 + + + + rental_date + TIMESTAMP + 19 + + + + inventory_id + INTEGER + 8 + + + + customer_id + SMALLINT + 5 + + + + return_date + TIMESTAMP + 19 + + + + staff_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_rental_customer + + + + + + + + + + + + + + fk_rental_inventory + + + + + + + + + + + + + + fk_rental_staff + + + + + + + + + + + + + + RENTAL_ID_NN + + + + + RENTAL_DATE_NN + + + + + INVENTORY_ID_NN + + + + + CUSTOMER_ID_NN + + + + + STAFF_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + rental_date + + + + + + + + + idx_fk_customer_id + OTHER + + + + + + + idx_fk_inventory_id + OTHER + + + + + + + idx_fk_staff_id + OTHER + + + false + + + + + + + + + + + + + + fk_payment_staff + + + + + + + + + + + + + + PAYMENT_ID_NN + + + + + CUSTOMER_ID_NN + + + + + STAFF_ID_NN + + + + + AMOUNT_NN + + + + + PAYMENT_DATE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + fk_payment_rental + OTHER + + + + + + + idx_fk_customer_id + OTHER + + + + + + + idx_fk_staff_id + OTHER + + + false +
+ +
+
+ + \ No newline at end of file diff --git a/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/GraphTest.java b/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/GraphTest.java new file mode 100644 index 00000000..3ba9a322 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/GraphTest.java @@ -0,0 +1,48 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.graph.Graph; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.graph.VertexRelationship; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.graph.Graph.Vertex; + +/** + * @类名: GraphTest + * @说明: 图测试 + * + * @author leehom + * @Date 2022年4月18日 上午10:50:36 + * 修改记录： + * + * @see + */ +public class GraphTest { + /** + 测试图： + a->b + ->c + d->e + f + */ + public static void main(String[] args) { + // 构建顶点 + Vertex va = new Vertex<>("a"); // 0 + Vertex vb = new Vertex<>("b"); // 1 + Vertex vc = new Vertex<>("c"); // 2 + Vertex vd = new Vertex<>("d"); // 3 + Vertex ve = new Vertex<>("e"); // 4 + Vertex vf = new Vertex<>("f"); // 5 + // 构建关系 + VertexRelationship vr1 = new VertexRelationship(0, 1); + VertexRelationship vr2 = new VertexRelationship(0, 2); + VertexRelationship vr3 = new VertexRelationship(3, 4); + // + Graph g = Graph.buildGraph(new Vertex[] {va,vb,vc,vd,ve,vf}, new VertexRelationship[] {vr1,vr2,vr3}); + // 打印 + g.print(); + + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/MysqlScannerTest.java b/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/MysqlScannerTest.java new file mode 100644 index 00000000..9a622ac3 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/MysqlScannerTest.java @@ -0,0 +1,89 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner; + +import org.junit.Before; +import org.junit.Test; + +import com.leehom.arch.datax.plugin.rdb2graph.common.BeanUtils; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.graph.Graph; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.BaseDataSourceBean; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.DruidConnectionProperties; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.DruidDataSourceBean; + + +/** + * @类名: MysqlScannerTest + * @说明: 数据库扫描引测试 + * + * @author leehom + * @Date 2012-8-29 下午5:55:46 + * + * + * @see + */ +public class MysqlScannerTest { + + private AbstractDbScanner mysqlScanner; + private DruidDataSourceBean dsBean; + +// db: sakila +// url: jdbc:mysql://localhost:3306/?remarks=true&useInformationSchema=false&serverTimezone=Asia/Shanghai +// tbs: +// username: root +// password: root +// driverClass: com.mysql.cj.jdbc.Driver +// filters: stat +// maxActive: 20 +// initialSize: 1 +// maxWait: 60000 +// minIdle: 1 +// timeBetweenEvictionRunsMillis: 60000 +// minEvictableIdleTimeMillis: 300000 +// validationQuery: select 'x' +// testWhileIdle: true +// testOnBorrow: false +// testOnReturn: false +// poolPreparedStatements: true +// maxOpenPreparedStatements: 20 + + @Before + public void init() { + dsBean = new DruidDataSourceBean(); + DruidConnectionProperties connProps = new DruidConnectionProperties(); + connProps.setDb("sakila"); + connProps.setUrl("jdbc:mysql://localhost:3306/?remarks=true&useInformationSchema=false&serverTimezone=Asia/Shanghai"); + connProps.setUserName("root"); + connProps.setPassword("123456"); + connProps.setDriverClass("com.mysql.jdbc.Driver"); + // + dsBean.setConnProps(connProps); + // + mysqlScanner = new MysqlScanner(); + mysqlScanner.setConnProps(connProps); + } + + @Test + public void testDbScan() throws Exception { + DbSchema dbmds = mysqlScanner.scan(dsBean); + BeanUtils.printBean(dbmds); + } + + @Test + public void testSetLinkTable() throws Exception { + DbSchema dbmds = mysqlScanner.scan(dsBean); + BeanUtils.printBean(dbmds); + } + + // 构建表连接图测试 + @Test + public void testBuildGraph() throws Exception { + DbSchema dbSch = mysqlScanner.scan(dsBean); + Graph g = dbSch.buildTableGraph(); + g.print(); + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/Neo4jDaoTest.java b/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/Neo4jDaoTest.java new file mode 100644 index 00000000..f2251c08 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/Neo4jDaoTest.java @@ -0,0 +1,115 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner; + +import java.text.MessageFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Test; +import org.neo4j.driver.Query; +import org.neo4j.driver.Record; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.leehom.arch.datax.plugin.rdb2graph.common.BeanUtils; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jDao; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jQueryPattern; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.ParamsUtils; + + + +/** + * @类名: Neo4jDaoTest + * @说明: 数据库扫描引测试 + * + * @author leehom + * @Date 2012-8-29 下午5:55:46 + * + * + * @see + */ +public class Neo4jDaoTest { + + private Neo4jDao neo4jDao; + + @Test + public void testCreateGraph() throws Exception { + // + Map params = new HashMap<>(); + params.put("dbname", "WCXX"); + Query query = new Query(Neo4jQueryPattern.CREATE_DB, params); + neo4jDao.executeQuery(query); + } + + @Test + public void testMatchALL() throws Exception { + // + Query query = new Query(Neo4jQueryPattern.MATCH_ALL); + List rs = neo4jDao.runQuery(query); + BeanUtils.printBean(rs); + } + + @Test + public void testShowIndex() throws Exception { + // + Query query = new Query("show indexes"); + List rs = neo4jDao.runQuery(query); + BeanUtils.printBean(rs); + } + + // 测试构建约束 + @Test + public void testCreateCONSTRAINT() throws Exception { + // + Map params = Maps.newHashMap(); + params.put("constraintname", "CONSTRAINT1"); + params.put("labelname", "yyy"); + // 索引字段 + params.put("properties", "n.x1, n.x2"); + Query query = new Query("CREATE CONSTRAINT constraint1 IF NOT EXISTS " + + "FOR (n:labelname) REQUIRE (n.x1, n.x2) IS UNIQUE", params); + neo4jDao.executeQuery(query); + + } + + @Test + public void testRunInTransaction() throws Exception { + String cql1 = MessageFormat.format(Neo4jQueryPattern.CREATE_NODE, "test", "{name: 'Andy1', title: 'Developer1'}"); + Query query1 = new Query(cql1); + + String cql2 = MessageFormat.format(Neo4jQueryPattern.CREATE_NODE, "test", "{name: 'Andy2', title: 'Developer2'}"); + Query query2 = new Query(cql2); + + neo4jDao.runInTransaction(Lists.newArrayList(query1, query2)); + } + + // + @Test + public void testRunInTransaction2() throws Exception { + // + Map params = Maps.newHashMap(); + params.put("key1", "xxxxx"); + params.put("key2", 123); + Date date = new Date(); + Instant instant = date.toInstant(); + ZoneId zoneId = ZoneId.systemDefault(); + LocalDateTime ldt = instant.atZone(zoneId).toLocalDateTime(); + params.put("key3", date); + // + String jsonProps = ParamsUtils.params2String(params); + + String cql1 = MessageFormat.format(Neo4jQueryPattern.CREATE_NODE, "test", jsonProps); + Query query1 = new Query(cql1);; + + neo4jDao.runInTransaction(Lists.newArrayList(query1)); + } + +} diff --git a/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/OracleScannerTest.java b/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/OracleScannerTest.java new file mode 100644 index 00000000..89ec3f64 --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/OracleScannerTest.java @@ -0,0 +1,75 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner; + +import java.io.File; +import java.io.InputStream; + +import org.junit.Test; + +import com.google.common.io.Files; +import com.leehom.arch.datax.plugin.rdb2graph.common.BeanUtils; +import com.leehom.arch.datax.plugin.rdb2graph.common.ByteAndStreamUtils; +import com.leehom.arch.datax.plugin.rdb2graph.common.ResourceLoaderUtil; +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.Serializer; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.BaseDataSourceBean; + + +/** + * @类名: OracleScannerTest + * @说明: oracle数据库扫描引测试 + * + * @author leehom + * @Date 2012-8-29 下午5:55:46 + * + * + * @see + */ +public class OracleScannerTest { + + private OracleScanner rdbScanner; + private BaseDataSourceBean dsBean; + private Serializer ser; + + @Test + public void testDbScan() throws Exception { + DbSchema dbmds = rdbScanner.scan(dsBean); + dbmds.buildTableGraph().print(); + } + + // 设置连接表 + @Test + public void testOracleRdbSchemaLinkTable() throws Exception { + // 载入schema + InputStream is = ResourceLoaderUtil.getResourceStream("wcc.xml"); + byte[] bytes = ByteAndStreamUtils.StreamToBytes(is); + DbSchema schema = (DbSchema)ser.Unmarshal(bytes); + // 连接表 + String partDocMaster = "WTPARTREFERENCELINK"; + String partDocMasterFk = "FK_IDA3A5_ID"; + TableMetadata table = schema.findTable(partDocMaster); + FKConstraintMetadata fkFC = table.findFk(partDocMasterFk); + schema.setLinkTable(table, fkFC); + // 序列化输出 + byte[] bytesLinkTable = ser.Marshal(schema); + File f = new File("wccx.xml"); + Files.write(bytesLinkTable, f); + + } + + @Test + public void testRdbSchemaDeSer() throws Exception { + // + InputStream is = ResourceLoaderUtil.getResourceStream("wccx.xml"); + byte[] bytes = ByteAndStreamUtils.StreamToBytes(is); + DbSchema schema = (DbSchema)ser.Unmarshal(bytes); + BeanUtils.printBeanDeep(schema); + } + + +} diff --git a/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/SchemaSerTest.java b/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/SchemaSerTest.java new file mode 100644 index 00000000..42266e5d --- /dev/null +++ b/rdb2graph/rdb2graph-scanner/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/scanner/SchemaSerTest.java @@ -0,0 +1,93 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.scanner; + +import java.io.File; + +import org.junit.Before; +import org.junit.Test; + +import com.google.common.io.Files; +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.Serializer; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.config.ScannerSerializerConfig; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.BaseDataSourceBean; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.DruidConnectionProperties; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.DruidDataSourceBean; + + +/** + * @类名: SchemaLoaderTest + * @说明: 数据库模式载入测试 + * + * @author leehom + * @Date 2012-8-29 下午5:55:46 + * + * + * @see + */ +public class SchemaSerTest { + + private Serializer ser; + private AbstractDbScanner dbScanner; + private DruidDataSourceBean dsBean; + + @Before + public void init() { + // + ser = ScannerSerializerConfig.rdbSchemaXmlSerializer(); + // + dsBean = new DruidDataSourceBean(); + DruidConnectionProperties connProps = new DruidConnectionProperties(); + connProps.setDb("sakila"); + connProps.setUrl("jdbc:mysql://localhost:3306/?remarks=true&useInformationSchema=false&serverTimezone=Asia/Shanghai"); + connProps.setUserName("root"); + connProps.setPassword("123456"); + connProps.setDriverClass("com.mysql.jdbc.Driver"); + // + dsBean.setConnProps(connProps); + // + dbScanner = new MysqlScanner(); + dbScanner.setConnProps(connProps); + } + + @Test + public void testRdbSchemaSer() throws Exception { + // + DbSchema schema = dbScanner.scan(dsBean); + byte[] bytes = ser.Marshal(schema); + File f = new File("sakila.xml"); + Files.write(bytes, f); + + } + + // 设置连接表 + // mysql + @Test + public void testMysqlRdbSchemaLinkTable() throws Exception { + // + DbSchema schema = dbScanner.scan(dsBean); + // 连接表 + String filmCategory = "film_category"; + String filmCategoryFk = "fk_film_category_film"; + TableMetadata tFilmCategory = schema.findTable(filmCategory); + FKConstraintMetadata fkFC = tFilmCategory.findFk(filmCategoryFk); + schema.setLinkTable(tFilmCategory, fkFC); + // + String filmActor = "film_actor"; + String filmActorFk = "fk_film_actor_film"; + TableMetadata tFilmActor = schema.findTable(filmActor); + FKConstraintMetadata fkFA = tFilmActor.findFk(filmActorFk); + schema.setLinkTable(tFilmActor, fkFA); + // + byte[] bytes = ser.Marshal(schema); + File f = new File("sakila.xml"); + Files.write(bytes, f); + + } + +} diff --git a/rdb2graph/rdb2graph-transformer/README.md b/rdb2graph/rdb2graph-transformer/README.md new file mode 100644 index 00000000..e69de29b diff --git a/rdb2graph/rdb2graph-transformer/pom.xml b/rdb2graph/rdb2graph-transformer/pom.xml new file mode 100644 index 00000000..17f12894 --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/pom.xml @@ -0,0 +1,57 @@ + + 4.0.0 + + com.leehom.arch.datax.plugin + rdb2graph-parent + ${revision} + + rdb2graph-transformer + jar + + + 0.9.0 + UTF-8 + + + + + + com.google.guava + guava + + + + org.projectlombok + lombok + + + + junit + junit + test + + + + com.leehom.arch.datax.plugin + rdb2graph-scanner + ${project.version} + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + logback.xml + application.yml + + + + + + diff --git a/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/SchemaTransformer.java b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/SchemaTransformer.java new file mode 100644 index 00000000..194061ac --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/SchemaTransformer.java @@ -0,0 +1,174 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.transformer; + +import java.util.List; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.AbstractDbScanner; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.graph.Graph.Vertex; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.graph.VertexRelationship; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jDao; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryWrapper; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.NotNullConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.UniqueConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKField; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.BaseDataSourceBean; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.IndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.NodeIndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.RelIndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.MappingRepository; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.SchemaMapping; + +import lombok.Data; + +/** + * @类名: SchemaTransformer + * @说明: 模式转换器 + * + * @author leehom + * @Date 2022年4月21日 下午6:40:58 + * 修改记录： + * + * @see + * + */ +@Data +public class SchemaTransformer { + + private AbstractDbScanner dbScanner; + private MappingRepository mappingRepository; + private Neo4jDao neo4jDao; + + /** + * @说明：转换模式(schema) + * 1. 数据库 + * 2. 表构件，按表连接图 + * 2.1 约束 唯一/非空 + * 2.2 索引 btree + * + * @author leehom + * @param dbSchema + * + */ + public void transform(BaseDataSourceBean dsBean) throws Exception { + DbSchema schema = dbScanner.scan(dsBean); + this.transform(schema); + } + public void transform(DbSchema dbSchem) { + Vertex[] vxs = new Vertex[dbSchem.getTables().size()]; + // 顶点，邻接点 + for(int i=0;i(dbSchem.getTables().get(i)); + } + // 构建连接表 + for(int i=0;i fks = table.getFks(); + // 顶点点 + if(fks.size()==0) { + this.transformVertex(dbSchem, vxs[i]); + continue; + } + // 连接表 + if(table.isLinkTable()) { + transformLinkRelationship(dbSchem, vxs[i]); + continue; + } + // 顶点/关系 + int from = i; + for(FKConstraintMetadata fk : fks) { + // + int to = locate(fk.getRefTable().getName(), vxs); + // 转换关系 + VertexRelationship> vr = new VertexRelationship>(from, to, fk.getFkFields()); + transformVertexRelationship(dbSchem, vxs, vr); + } + } + + } + + // 转换顶点 + public void transformVertex(DbSchema dbSchema, Vertex v) { + // 唯一约束 + TableMetadata tbmd = v.getData(); + List uqcs = tbmd.getUnique(); + for(UniqueConstraintMetadata uqc : uqcs) { + SchemaMapping mapping = mappingRepository.getMapping(UniqueConstraintMetadata.class); + QueryWrapper query = mapping.mapTo(dbSchema, uqc); + neo4jDao.executeQuery(query.getQuery()); + } + // 非空约束 + List nncs = tbmd.getNotNull(); + for(NotNullConstraintMetadata nnc : nncs) { + SchemaMapping mapping = mappingRepository.getMapping(NotNullConstraintMetadata.class); + QueryWrapper query = mapping.mapTo(dbSchema, nnc); + neo4jDao.executeQuery(query.getQuery()); + } + // 索引 + List indexmds = tbmd.getIndexes(); + for(IndexMetadata index : indexmds) { + SchemaMapping mapping = mappingRepository.getMapping(NodeIndexMetadata.class); + QueryWrapper query = mapping.mapTo(dbSchema, new NodeIndexMetadata(index)); + neo4jDao.executeQuery(query.getQuery()); + } + } + // 转换连接表关系，node->edge->edge + public void transformLinkRelationship(DbSchema dbSchema, Vertex v) { + // 唯一约束 + TableMetadata tbmd = v.getData(); + List uqcs = tbmd.getUnique(); + for (UniqueConstraintMetadata uqc : uqcs) { + SchemaMapping mapping = mappingRepository.getMapping(UniqueConstraintMetadata.class); + QueryWrapper query = mapping.mapTo(dbSchema, uqc); + neo4jDao.executeQuery(query.getQuery()); + } + // 非空约束 + List nncs = tbmd.getNotNull(); + for (NotNullConstraintMetadata nnc : nncs) { + SchemaMapping mapping = mappingRepository.getMapping(NotNullConstraintMetadata.class); + QueryWrapper query = mapping.mapTo(dbSchema, nnc); + neo4jDao.executeQuery(query.getQuery()); + } + // 索引，按关系索引 + List indexmds = tbmd.getIndexes(); + for (IndexMetadata index : indexmds) { + SchemaMapping mapping = mappingRepository.getMapping(RelIndexMetadata.class); + QueryWrapper query = mapping.mapTo(dbSchema, new RelIndexMetadata(index)); + neo4jDao.executeQuery(query.getQuery()); + } + + } + // 转换顶点/关系，node->edge->node + public void transformVertexRelationship(DbSchema dbSchema, Vertex[] vxs, VertexRelationship> vr) { + // 左顶点 + Vertex left = vxs[vr.getFrom()]; + transformVertex(dbSchema, left); + // 右顶点 + Vertex right = vxs[vr.getTo()]; + transformVertex(dbSchema, right); + // 关系 + this.transformRelationship(dbSchema, vr); + } + // 转换关系 + public void transformRelationship(DbSchema dbSchema, VertexRelationship> vr) { + + } + + // 获取表邻接数组位置 + private int locate(String tableName, Vertex[] vxs) { + int l = 0; + for(Vertex vx : vxs) { + if(tableName.equals(vx.getData().getName())) + break; + l++; + } + return l; + + } +} diff --git a/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/StringUtils.java b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/StringUtils.java new file mode 100644 index 00000000..f4444ffd --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/StringUtils.java @@ -0,0 +1,26 @@ +package com.leehom.arch.datax.plugin.rdb2graph.transformer; + +import java.util.List; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.FieldMetadata; + +public class StringUtils { + + /** */ + public static final String FIELD_SEQ = ","; + + // 字段名排列 + public static String fieldNames2String(String prefix, List fields) { + if (fields == null || fields.size() == 0) + return ""; + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < fields.size(); i++) { + FieldMetadata f = fields.get(i); + sb.append(prefix+f.getName()); + if (i != fields.size() - 1) + sb.append(FIELD_SEQ); + } + return sb.toString(); + } + +} diff --git a/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/config/SchemaMappingBeanConfig.java b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/config/SchemaMappingBeanConfig.java new file mode 100644 index 00000000..8ef1f7e9 --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/config/SchemaMappingBeanConfig.java @@ -0,0 +1,51 @@ +/** + * + */ +package com.leehom.arch.datax.plugin.rdb2graph.transformer.config; + +import java.util.HashMap; +import java.util.Map; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.NotNullConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.UniqueConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.NodeIndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.RelIndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.MappingRepository; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.SchemaMapping; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.support.NodeIndexMapping; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.support.NotNullConstraintMapping; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.support.RelIndexMapping; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.support.UniqueConstraintMapping; + +/** + * @类名: SchemaMappingBeanConfig + * @说明: 搜索映射bean配置 + * + * @author leehom + * @Date 2019年10月8日 下午4:21:34 + * 修改记录： + * + * @see + */ +public class SchemaMappingBeanConfig { + + public static MappingRepository mappingRepository() { + MappingRepository mappingRepo = new MappingRepository(); + // + Map mappings = new HashMap<>(); + // 创建数据库屏蔽 + // mappings.put(DbSchema.class, new DbSchemaMapping()); + // 创建唯一约束 + mappings.put(UniqueConstraintMetadata.class, new UniqueConstraintMapping()); + // 创建非空约束 + mappings.put(NotNullConstraintMetadata.class, new NotNullConstraintMapping()); + // 索引 + mappings.put(NodeIndexMetadata.class, new NodeIndexMapping()); + mappings.put(RelIndexMetadata.class, new RelIndexMapping()); + // + mappingRepo.setMappings(mappings); + // + return mappingRepo; + } + +} diff --git a/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/MappingRepository.java b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/MappingRepository.java new file mode 100644 index 00000000..b7f5e776 --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/MappingRepository.java @@ -0,0 +1,38 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping; + +import java.util.Map; + +/** + * @类名: MappingRepository + * @说明: + * + * @author leehom + * @Date 2022年4月21日 下午6:44:11 + * 修改记录： + * + * @see + */ +public class MappingRepository { + + /** 映射子子映射实现*/ + private Map mappings; + + /** + * 说明：获取合适的映射实现 + * @author leehom + * @param op + * @return + * + */ + public SchemaMapping getMapping(Class clazz) { + return mappings.get(clazz); + } + + public void setMappings(Map mappings) { + this.mappings = mappings; + } +} diff --git a/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/SchemaMapping.java b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/SchemaMapping.java new file mode 100644 index 00000000..5f6322f8 --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/SchemaMapping.java @@ -0,0 +1,33 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryWrapper; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; + +/** + * @类名: SchemaMapping + * @说明: schema构件映射 + * + * @author leehom + * @Date 2022年4月21日 下午6:52:43 + * 修改记录： + * + * @see + */ +public interface SchemaMapping { + + /** + * 说明：映射rdb模式构件，生成neo4j query + * + * @author leehom + * @param op + * @param sch 索引特性 + * @return + * + */ + public QueryWrapper mapTo(DbSchema dbSchema, T schemaItem); + +} diff --git a/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/DbSchemaMapping.java b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/DbSchemaMapping.java new file mode 100644 index 00000000..3e276c1d --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/DbSchemaMapping.java @@ -0,0 +1,40 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.support; + +import java.util.Map; + +import org.neo4j.driver.Query; + +import com.google.common.collect.Maps; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jQueryPattern; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryType; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryWrapper; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.SchemaMapping; + +/** + * @类名: DbSchemaMapping + * @说明: 数据库映射 + * + * @author leehom + * @Date 2022年4月22日 下午4:43:58 + * 修改记录： + * + * @see + */ +public class DbSchemaMapping implements SchemaMapping { + + @Override + public QueryWrapper mapTo(DbSchema dbSchema, DbSchema schemaItem) { + // + String dbName = dbSchema.getName(); + Map params = Maps.newHashMap(); + params.put("dbname", dbName); + Query query = new Query(Neo4jQueryPattern.CREATE_DB, params); + return QueryWrapper.wrap(query, QueryType.INSERT); + } + +} diff --git a/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/NodeIndexMapping.java b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/NodeIndexMapping.java new file mode 100644 index 00000000..ca1be5db --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/NodeIndexMapping.java @@ -0,0 +1,44 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.support; + +import java.text.MessageFormat; + +import org.neo4j.driver.Query; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jQueryPattern; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryType; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryWrapper; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.IndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.NodeIndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.StringUtils; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.SchemaMapping; + +/** + * @类名: BTreeIndexMapping + * @说明: 唯一约束映射 + * + * @author leehom + * @Date 2022年4月22日 下午4:43:58 + * 修改记录： + * + * @see + */ +public class NodeIndexMapping implements SchemaMapping { + + @Override + public QueryWrapper mapTo(DbSchema dbSchema, NodeIndexMetadata nindex) { + IndexMetadata index = nindex.getIndexmd(); + // + String cql = MessageFormat.format(Neo4jQueryPattern.CREATE_NODE_INDEX, + index.getName(), + index.getTbmd().getName(), // label + StringUtils.fieldNames2String(Neo4jQueryPattern.PROPERTTY_ALIAS, index.getFields())); + Query query = new Query(cql); + return QueryWrapper.wrap(query, QueryType.INSERT); + } + +} diff --git a/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/NotNullConstraintMapping.java b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/NotNullConstraintMapping.java new file mode 100644 index 00000000..c1b0c0e7 --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/NotNullConstraintMapping.java @@ -0,0 +1,42 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.support; + +import java.text.MessageFormat; + +import org.neo4j.driver.Query; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jQueryPattern; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryType; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryWrapper; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.NotNullConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.SchemaMapping; + +/** + * @类名: UniqueConstraintMapping + * @说明: 唯一约束映射 + * + * @author leehom + * @Date 2022年4月22日 下午4:43:58 + * 修改记录： + * + * @see + */ +public class NotNullConstraintMapping implements SchemaMapping { + + @Override + public QueryWrapper mapTo(DbSchema dbSchema, NotNullConstraintMetadata nnc) { + // + String cql = MessageFormat.format(Neo4jQueryPattern.CREATE_NN_CONSTRAINT, + nnc.getNnName(), + nnc.getTbmd().getName(), // label + Neo4jQueryPattern.PROPERTTY_ALIAS+nnc.getField().getName() + ); + Query query = new Query(cql); + return QueryWrapper.wrap(query, QueryType.INSERT); + } + +} diff --git a/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/RelIndexMapping.java b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/RelIndexMapping.java new file mode 100644 index 00000000..c55f97b0 --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/RelIndexMapping.java @@ -0,0 +1,44 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.support; + +import java.text.MessageFormat; + +import org.neo4j.driver.Query; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jQueryPattern; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryType; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryWrapper; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.IndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.index.RelIndexMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.StringUtils; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.SchemaMapping; + +/** + * @类名: BTreeIndexMapping + * @说明: 唯一约束映射 + * + * @author leehom + * @Date 2022年4月22日 下午4:43:58 + * 修改记录： + * + * @see + */ +public class RelIndexMapping implements SchemaMapping { + + @Override + public QueryWrapper mapTo(DbSchema dbSchema, RelIndexMetadata nindex) { + IndexMetadata index = nindex.getIndexmd(); + // + String cql = MessageFormat.format(Neo4jQueryPattern.CREATE_REL_INDEX, + index.getName(), + index.getTbmd().getName(), // label + StringUtils.fieldNames2String(Neo4jQueryPattern.PROPERTTY_ALIAS, index.getFields())); + Query query = new Query(cql); + return QueryWrapper.wrap(query, QueryType.INSERT); + } + +} diff --git a/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/UniqueConstraintMapping.java b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/UniqueConstraintMapping.java new file mode 100644 index 00000000..1924eaef --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/mapping/support/UniqueConstraintMapping.java @@ -0,0 +1,41 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.support; + +import java.text.MessageFormat; + +import org.neo4j.driver.Query; + +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jQueryPattern; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryType; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.QueryWrapper; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.UniqueConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.StringUtils; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.mapping.SchemaMapping; + +/** + * @类名: UniqueConstraintMapping + * @说明: 唯一约束映射 + * + * @author leehom + * @Date 2022年4月22日 下午4:43:58 + * 修改记录： + * + * @see + */ +public class UniqueConstraintMapping implements SchemaMapping { + + @Override + public QueryWrapper mapTo(DbSchema dbSchema, UniqueConstraintMetadata unq) { + // + String cql = MessageFormat.format(Neo4jQueryPattern.CREATE_UNQ_CONSTRAINT, + unq.getUnqName(), unq.getTbmd().getName(), + StringUtils.fieldNames2String(Neo4jQueryPattern.PROPERTTY_ALIAS, unq.getFields())); + Query query = new Query(cql); + return QueryWrapper.wrap(query, QueryType.INSERT); + } + +} diff --git a/rdb2graph/rdb2graph-transformer/src/main/resources/application-dev.yml b/rdb2graph/rdb2graph-transformer/src/main/resources/application-dev.yml new file mode 100644 index 00000000..0a5ae6d0 --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/resources/application-dev.yml @@ -0,0 +1,37 @@ +spring: + application: + name: rdb2graph-transformer + +setl: + scanner: + type: oracle + serializer: # 序列化 + type: xml + ds: + db: WC + url: jdbc:oracle:thin:@192.168.4.126:1521:wcc + tbs: # filter table + username: wc + password: wc + driverClass: oracle.jdbc.driver.OracleDriver + filters: stat + maxActive: 20 + initialSize: 1 + maxWait: 60000 + minIdle: 1 + timeBetweenEvictionRunsMillis: 60000 + minEvictableIdleTimeMillis: 300000 + validationQuery: select 'x' from dual + testWhileIdle: true + testOnBorrow: false + testOnReturn: false + poolPreparedStatements: true + maxOpenPreparedStatements: 20 + +org: + neo4j: + driver: + uri: bolt://34.201.15.62:7687 + authentication: + username: neo4j + password: relay-accomplishment-mover \ No newline at end of file diff --git a/rdb2graph/rdb2graph-transformer/src/main/resources/application-local.yml b/rdb2graph/rdb2graph-transformer/src/main/resources/application-local.yml new file mode 100644 index 00000000..bc246fbc --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/resources/application-local.yml @@ -0,0 +1,37 @@ +spring: + application: + name: rdb2graph-transformer + +setl: + scanner: + type: mysql + serializer: # 序列化 + type: xml + ds: + db: sakila + url: jdbc:mysql://localhost:3306?remarks=true&useInformationSchema=false&serverTimezone=Asia/Shanghai + tbs: + ltbs: # 连接表 + username: root + password: root + driverClass: com.mysql.cj.jdbc.Driver + filters: stat + maxActive: 20 + initialSize: 1 + maxWait: 60000 + minIdle: 1 + timeBetweenEvictionRunsMillis: 60000 + minEvictableIdleTimeMillis: 300000 + validationQuery: select 'x' + testWhileIdle: true + testOnBorrow: false + testOnReturn: false + poolPreparedStatements: true + maxOpenPreparedStatements: 20 + +org: + neo4j: + driver: + uri: bolt://192.168.1.20:7687 + username: neo4j + password: neo4j1234 \ No newline at end of file diff --git a/rdb2graph/rdb2graph-transformer/src/main/resources/application.yml b/rdb2graph/rdb2graph-transformer/src/main/resources/application.yml new file mode 100644 index 00000000..d74c444c --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/resources/application.yml @@ -0,0 +1,3 @@ +spring: + profiles: + active: local diff --git a/rdb2graph/rdb2graph-transformer/src/main/resources/logback.xml b/rdb2graph/rdb2graph-transformer/src/main/resources/logback.xml new file mode 100644 index 00000000..0f648972 --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/resources/logback.xml @@ -0,0 +1,76 @@ + + + + spring-boot-log + + + + + + [%date] %clr([%level]) %clr([%logger]:%L) >>> %msg %n + utf-8 + + + + INFO + + + + + logs/info.log + + + + logs/info-%d{yyyy-MM-dd}.%i.log + + 128MB + + + 30 + + true + + [%date] %clr([%level]) %clr([%logger]:%L) >>> %msg %n + utf-8 + + + info + ACCEPT + DENY + + + + + logs/error.log + + + + logs/error-%d{yyyy-MM-dd}.%i.log + + 2MB + + + 180 + + true + + + [%date] %clr([%level]) [%thread] %clr([%logger]:%L) >>> %msg %n + utf-8 + + + + + ERROR + + ACCEPT + + DENY + + + + + + + + diff --git a/rdb2graph/rdb2graph-transformer/src/main/resources/sakila.xml b/rdb2graph/rdb2graph-transformer/src/main/resources/sakila.xml new file mode 100644 index 00000000..65ef1eaa --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/main/resources/sakila.xml @@ -0,0 +1,1849 @@ + + sakila + +
+ actor + + + actor_id + SMALLINT + 5 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + ACTOR_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_actor_last_name + OTHER + + + false +
+ + address + + + address_id + SMALLINT + 5 + + + + address + VARCHAR + 50 + + + + address2 + VARCHAR + 50 + + + + district + VARCHAR + 20 + + + + city_id + SMALLINT + 5 + + + + postal_code + VARCHAR + 10 + + + + phone + VARCHAR + 20 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_address_city + + city + + + city_id + SMALLINT + 5 + + + + city + VARCHAR + 50 + + + + country_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_city_country + + country + + + country_id + SMALLINT + 5 + + + + country + VARCHAR + 50 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + COUNTRY_ID_NN + + + + + COUNTRY_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false + + + + + + + + + + + + + + CITY_ID_NN + + + + + CITY_NN + + + + + COUNTRY_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_country_id + OTHER + + + false + + + + + + + + + + + + + + ADDRESS_ID_NN + + + + + ADDRESS_NN + + + + + DISTRICT_NN + + + + + CITY_ID_NN + + + + + PHONE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_city_id + OTHER + + + false +
+ + category + + + category_id + TINYINT + 3 + + + + name + VARCHAR + 25 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + CATEGORY_ID_NN + + + + + NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false +
+ +
+
+ customer + + + customer_id + SMALLINT + 5 + + + + store_id + TINYINT + 3 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + email + VARCHAR + 50 + + + + address_id + SMALLINT + 5 + + + + active + BIT + 0 + + + + create_date + TIMESTAMP + 19 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_customer_address + + + + + + + + + + + + + + fk_customer_store + + store + + + store_id + TINYINT + 3 + + + + manager_staff_id + TINYINT + 3 + + + + address_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_store_address + + + + + + + + + + + + + + fk_store_staff + + staff + + + staff_id + TINYINT + 3 + + + + first_name + VARCHAR + 45 + + + + last_name + VARCHAR + 45 + + + + address_id + SMALLINT + 5 + + + + picture + LONGVARBINARY + 65535 + + + + email + VARCHAR + 50 + + + + store_id + TINYINT + 3 + + + + active + BIT + 0 + + + + username + VARCHAR + 16 + + + + password + VARCHAR + 40 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_staff_address + + + + + + + + + + + + + + fk_staff_store + + + + + + + + + + + + + + STAFF_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + ADDRESS_ID_NN + + + + + STORE_ID_NN + + + + + ACTIVE_NN + + + + + USERNAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + + + + + idx_fk_store_id + OTHER + + + false + + + + + + + + + + + + + + STORE_ID_NN + + + + + MANAGER_STAFF_ID_NN + + + + + ADDRESS_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + idx_unique_manager + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + false + + + + + + + + + + + + + + CUSTOMER_ID_NN + + + + + STORE_ID_NN + + + + + FIRST_NAME_NN + + + + + LAST_NAME_NN + + + + + ADDRESS_ID_NN + + + + + ACTIVE_NN + + + + + CREATE_DATE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_address_id + OTHER + + + + + + + idx_fk_store_id + OTHER + + + + + + + idx_last_name + OTHER + + + false +
+ + film + + + film_id + SMALLINT + 5 + + + + title + VARCHAR + 128 + + + + description + LONGVARCHAR + 65535 + + + + release_year + DATE + 0 + + + + language_id + TINYINT + 3 + + + + original_language_id + TINYINT + 3 + + + + rental_duration + TINYINT + 3 + + + + rental_rate + DECIMAL + 4 + + + + length + SMALLINT + 5 + + + + replacement_cost + DECIMAL + 5 + + + + rating + CHAR + 5 + + + + special_features + CHAR + 54 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_film_language + + language + + + language_id + TINYINT + 3 + + + + name + CHAR + 20 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + LANGUAGE_ID_NN + + + + + NAME_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + false + + + + + + + + + + + + + + fk_film_language_original + + + + + + + + + + + + + + FILM_ID_NN + + + + + TITLE_NN + + + + + LANGUAGE_ID_NN + + + + + RENTAL_DURATION_NN + + + + + RENTAL_RATE_NN + + + + + REPLACEMENT_COST_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_language_id + OTHER + + + + + + + idx_fk_original_language_id + OTHER + + + + + + + idx_title + OTHER + + + false +
+ + film_actor + + + actor_id + SMALLINT + 5 + + + + film_id + SMALLINT + 5 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + + fk_film_actor_actor + + + + + + + + + + + + + + fk_film_actor_film + + + + + + + + + + + + + + ACTOR_ID_NN + + + + + FILM_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_film_id + OTHER + + + true + +
+ + film_category + + + film_id + SMALLINT + 5 + + + + category_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + + fk_film_category_category + + + + + + + + + + + + + + fk_film_category_film + + + + + + + + + + + + + + FILM_ID_NN + + + + + CATEGORY_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + + PRIMARY + + + + + + + + + fk_film_category_category + OTHER + + + true + +
+ + film_text + + + film_id + SMALLINT + 5 + + + + title + VARCHAR + 255 + + + + description + LONGVARCHAR + 65535 + + + + + + + + + + + + + + + FILM_ID_NN + + + + + TITLE_NN + + + + + + + + + PRIMARY + + + + + + + + + + idx_title_description + OTHER + + + false +
+ + inventory + + + inventory_id + INTEGER + 8 + + + + film_id + SMALLINT + 5 + + + + store_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_inventory_film + + + + + + + + + + + + + + fk_inventory_store + + + + + + + + + + + + + + INVENTORY_ID_NN + + + + + FILM_ID_NN + + + + + STORE_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + idx_fk_film_id + OTHER + + + + + + + + idx_store_id_film_id + OTHER + + + false +
+ +
+ payment + + + payment_id + SMALLINT + 5 + + + + customer_id + SMALLINT + 5 + + + + staff_id + TINYINT + 3 + + + + rental_id + INTEGER + 10 + + + + amount + DECIMAL + 5 + + + + payment_date + TIMESTAMP + 19 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_payment_customer + + + + + + + + + + + + + + fk_payment_rental + + rental + + + rental_id + INTEGER + 10 + + + + rental_date + TIMESTAMP + 19 + + + + inventory_id + INTEGER + 8 + + + + customer_id + SMALLINT + 5 + + + + return_date + TIMESTAMP + 19 + + + + staff_id + TINYINT + 3 + + + + last_update + TIMESTAMP + 19 + + + + + + + + + + + + + + + + fk_rental_customer + + + + + + + + + + + + + + fk_rental_inventory + + + + + + + + + + + + + + fk_rental_staff + + + + + + + + + + + + + + RENTAL_ID_NN + + + + + RENTAL_DATE_NN + + + + + INVENTORY_ID_NN + + + + + CUSTOMER_ID_NN + + + + + STAFF_ID_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + rental_date + + + + + + + + + idx_fk_customer_id + OTHER + + + + + + + idx_fk_inventory_id + OTHER + + + + + + + idx_fk_staff_id + OTHER + + + false + + + + + + + + + + + + + + fk_payment_staff + + + + + + + + + + + + + + PAYMENT_ID_NN + + + + + CUSTOMER_ID_NN + + + + + STAFF_ID_NN + + + + + AMOUNT_NN + + + + + PAYMENT_DATE_NN + + + + + LAST_UPDATE_NN + + + + + + + + + PRIMARY + + + + + + + + + fk_payment_rental + OTHER + + + + + + + idx_fk_customer_id + OTHER + + + + + + + idx_fk_staff_id + OTHER + + + false +
+ +
+
+ + \ No newline at end of file diff --git a/rdb2graph/rdb2graph-transformer/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/SchemaTransformerTest.java b/rdb2graph/rdb2graph-transformer/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/SchemaTransformerTest.java new file mode 100644 index 00000000..3fdeae48 --- /dev/null +++ b/rdb2graph/rdb2graph-transformer/src/test/java/com/leehom/arch/datax/plugin/rdb2graph/transformer/SchemaTransformerTest.java @@ -0,0 +1,90 @@ +/** + * %datax-graph% + * %v1.0% + */ +package com.leehom.arch.datax.plugin.rdb2graph.transformer; + +import java.io.InputStream; +import java.net.URI; + +import org.junit.Before; +import org.junit.Test; + +import com.leehom.arch.datax.plugin.rdb2graph.common.ByteAndStreamUtils; +import com.leehom.arch.datax.plugin.rdb2graph.common.ResourceLoaderUtil; +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.Serializer; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.AbstractDbScanner; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.MysqlScanner; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.config.Neo4jDaoConfiguration; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.config.Neo4jDriverProperties; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.config.ScannerSerializerConfig; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.neo4j.ds.Neo4jDao; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.DruidConnectionProperties; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.ds.DruidDataSourceBean; +import com.leehom.arch.datax.plugin.rdb2graph.transformer.config.SchemaMappingBeanConfig; + + +/** + * @类名: Neo4jDaoTest + * @说明: 数据库扫描引测试 + * + * @author leehom + * @Date 2012-8-29 下午5:55:46 + * + * + * @see + */ +public class SchemaTransformerTest { + + private SchemaTransformer transformer; + private DruidDataSourceBean dsBean; + private Serializer ser; + private AbstractDbScanner dbScanner; + private Neo4jDriverProperties props; + private Neo4jDao neo4jDao; + + @Before + public void init() { + // + ser = ScannerSerializerConfig.rdbSchemaXmlSerializer(); + // + dsBean = new DruidDataSourceBean(); + DruidConnectionProperties connProps = new DruidConnectionProperties(); + connProps.setDb("sakila"); + connProps.setUrl("jdbc:mysql://localhost:3306/?remarks=true&useInformationSchema=false&serverTimezone=Asia/Shanghai"); + connProps.setUserName("root"); + connProps.setPassword("123456"); + connProps.setDriverClass("com.mysql.jdbc.Driver"); + dsBean.setConnProps(connProps); + // + dbScanner = new MysqlScanner(); + dbScanner.setConnProps(connProps); + // + props = new Neo4jDriverProperties(); + props.setDatabase("sakila"); + props.setUri(URI.create("bolt://192.168.1.20:7687")); + props.setUsername("neo4j"); + props.setPassword("neo4j1234"); + neo4jDao = Neo4jDaoConfiguration.neo4jDao(props); + // + transformer = new SchemaTransformer(); + transformer.setDbScanner(dbScanner); + transformer.setMappingRepository(SchemaMappingBeanConfig.mappingRepository()); + transformer.setNeo4jDao(neo4jDao); + } + + @Test + public void testTransform() throws Exception { + transformer.transform(dsBean); + } + + + @Test + public void testTransformScheam() throws Exception { + InputStream is = ResourceLoaderUtil.getResourceStream("sakila.xml"); + byte[] bytes = ByteAndStreamUtils.StreamToBytes(is); + DbSchema schema = (DbSchema)ser.Unmarshal(bytes); + transformer.transform(schema); + } +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/pom.xml b/rdb2graph/rdb2grpah-rdbms-util/pom.xml new file mode 100644 index 00000000..213e56ce --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/pom.xml @@ -0,0 +1,65 @@ + + 4.0.0 + + com.leehom.arch.datax.plugin + rdb2graph-parent + ${revision} + + rdb2graph-rdbms-util + jar + + + + com.alibaba.datax + datax-common + + + slf4j-log4j12 + org.slf4j + + + + + commons-collections + commons-collections + 3.0 + + + mysql + mysql-connector-java + ${mysql.driver.version} + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + com.alibaba + druid + + + junit + junit + test + + + com.google.guava + guava + + + com.leehom.arch.datax.plugin + rdb2graph-scanner + ${revision} + + + com.leehom.arch.datax.plugin + rdb2graph-common + ${revision} + + + diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/CommonRdbms2GraphReader.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/CommonRdbms2GraphReader.java new file mode 100644 index 00000000..e517cf22 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/CommonRdbms2GraphReader.java @@ -0,0 +1,378 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader; + +import java.io.InputStream; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.datax.common.element.BoolColumn; +import com.alibaba.datax.common.element.BytesColumn; +import com.alibaba.datax.common.element.DateColumn; +import com.alibaba.datax.common.element.DoubleColumn; +import com.alibaba.datax.common.element.LongColumn; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.element.StringColumn; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.alibaba.datax.common.statistics.PerfRecord; +import com.alibaba.datax.common.statistics.PerfTrace; +import com.alibaba.datax.common.util.Configuration; +import com.google.common.collect.Lists; +import com.leehom.arch.datax.plugin.rdb2graph.common.ByteAndStreamUtils; +import com.leehom.arch.datax.plugin.rdb2graph.common.ResourceLoaderUtil; +import com.leehom.arch.datax.plugin.rdb2graph.common.serializer.Serializer; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.util.OriginalConfPretreatmentUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.util.PreCheckTask; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.util.ReaderSplitUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.util.SingleTableSplitUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtilErrorCode; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DataBaseType; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.RdbmsException; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.config.ScannerSerializerConfig; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; + +/** + * @类名: CommonRdbms2GraphReader + * @说明: rdb2graph读入器 + * + * @author leehom + * @Date 2022年4月29日 下午1:48:19 + * 修改记录： + * + * @see + */ +public class CommonRdbms2GraphReader { + + public static class Job { + private static final Logger LOG = LoggerFactory.getLogger(Job.class); + + private Serializer ser; + private DbSchema rdbSchema; + + public Job(DataBaseType dataBaseType) { + OriginalConfPretreatmentUtil.DATABASE_TYPE = dataBaseType; + SingleTableSplitUtil.DATABASE_TYPE = dataBaseType; + } + + public void init(Configuration originalConfig) { + // 载入schema + // 初始化序列化器 + ser = ScannerSerializerConfig.rdbSchemaXmlSerializer(); + // + String schemaUri = originalConfig.getString(Constant.RDB_SCHEMA_URI); + try (InputStream is = ResourceLoaderUtil.getResourceStream(schemaUri);){ + byte[] bytes = ByteAndStreamUtils.StreamToBytes(is); + rdbSchema = (DbSchema)ser.Unmarshal(bytes); + } catch (Exception e) { + throw DataXException.asDataXException(DBUtilErrorCode.ERROR_LOAD_RDBSCHEMA, e); + } + // 原生配置预处理 + OriginalConfPretreatmentUtil.doPretreatment(originalConfig, rdbSchema); + LOG.debug("After job init(), job config now is:[\n{}\n]", originalConfig.toJSON()); + } + + // + public void preCheck(Configuration originalConfig, DataBaseType dataBaseType) { + /* 检查每个表是否有读权限，以及querySql跟splik Key是否正确 */ + Configuration queryConf = ReaderSplitUtil.doPreCheckSplit(originalConfig); + String splitPK = queryConf.getString(Key.SPLIT_PK); + List connList = queryConf.getList(Constant.CONN_MARK, Object.class); + String username = queryConf.getString(Key.USERNAME); + String password = queryConf.getString(Key.PASSWORD); + ExecutorService exec; + if (connList.size() < 10) { + exec = Executors.newFixedThreadPool(connList.size()); + } else { + exec = Executors.newFixedThreadPool(10); + } + Collection taskList = new ArrayList(); + for (int i = 0, len = connList.size(); i < len; i++) { + Configuration connConf = Configuration.from(connList.get(i).toString()); + PreCheckTask t = new PreCheckTask(username, password, connConf, dataBaseType, splitPK); + taskList.add(t); + } + List> results = Lists.newArrayList(); + try { + results = exec.invokeAll(taskList); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + for (Future result : results) { + try { + result.get(); + } catch (ExecutionException e) { + DataXException de = (DataXException) e.getCause(); + throw de; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + exec.shutdownNow(); + } + + public List split(Configuration originalConfig, int adviceNumber) { + return ReaderSplitUtil.doSplit(originalConfig, adviceNumber); + } + + public void post(Configuration originalConfig) { + // do nothing + } + + public void destroy(Configuration originalConfig) { + // do nothing + } + + } + + //*******************任务******************* + public static class Task { + private static final Logger LOG = LoggerFactory.getLogger(Task.class); + private static final boolean IS_DEBUG = LOG.isDebugEnabled(); + protected final byte[] EMPTY_CHAR_ARRAY = new byte[0]; + + private DataBaseType dataBaseType; + private int taskGroupId = -1; + private int taskId = -1; + + private String username; + private String password; + private String jdbcUrl; + private String mandatoryEncoding; + private Phase phase; + + // 作为日志显示信息时，需要附带的通用信息。比如信息所对应的数据库连接等信息，针对哪个表做的操作 + private String basicMsg; + + public Task(DataBaseType dataBaseType) { + this(dataBaseType, -1, -1); + } + + public Task(DataBaseType dataBaseType, int taskGropuId, int taskId) { + this.dataBaseType = dataBaseType; + this.taskGroupId = taskGropuId; + this.taskId = taskId; + } + + public void init(Configuration readerSliceConfig) { + + this.username = readerSliceConfig.getString(Key.USERNAME); + this.password = readerSliceConfig.getString(Key.PASSWORD); + this.jdbcUrl = readerSliceConfig.getString(Key.JDBC_URL); + // + String phaseStr = readerSliceConfig.getString(Key.GRAPH_PHASE); + this.phase = Phase.valueOf(phaseStr.toUpperCase()); + // ob10的处理 + if (this.jdbcUrl.startsWith(com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.Constant.OB10_SPLIT_STRING) + && this.dataBaseType == DataBaseType.MySql) { + String[] ss = this.jdbcUrl + .split(com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.Constant.OB10_SPLIT_STRING_PATTERN); + if (ss.length != 3) { + throw DataXException.asDataXException(DBUtilErrorCode.JDBC_OB10_ADDRESS_ERROR, + "JDBC OB10格式错误，请联系askdatax"); + } + LOG.info("this is ob1_0 jdbc url."); + this.username = ss[1].trim() + ":" + this.username; + this.jdbcUrl = ss[2]; + LOG.info("this is ob1_0 jdbc url. user=" + this.username + " :url=" + this.jdbcUrl); + } + + this.mandatoryEncoding = readerSliceConfig.getString(Key.MANDATORY_ENCODING, ""); + + basicMsg = String.format("jdbcUrl:[%s]", this.jdbcUrl); + + // + + } + + // 读入数据 + public void startRead(Configuration readerSliceConfig, RecordSender recordSender, + TaskPluginCollector taskPluginCollector, int fetchSize) { + String querySql = readerSliceConfig.getString(Key.QUERY_SQL); + String table = readerSliceConfig.getString(Key.TABLE); + // 性能跟踪，注册任务 + PerfTrace.getInstance().addTaskDetails(taskId, table + "," + basicMsg); + // + LOG.info("Begin to read record by Sql: [{}\n] {}.", querySql, basicMsg); + // 查询性能记录开始 + PerfRecord queryPerfRecord = new PerfRecord(taskGroupId, taskId, PerfRecord.PHASE.SQL_QUERY); + queryPerfRecord.start(); + // 获取连接 + Connection conn = DBUtil.getConnection(this.dataBaseType, jdbcUrl, username, password); + // session config .etc related + DBUtil.dealWithSessionConfig(conn, readerSliceConfig, this.dataBaseType, basicMsg); + // + int columnNumber = 0; + ResultSet rs = null; + try { + // 执行查询 + rs = DBUtil.query(conn, querySql, fetchSize); + queryPerfRecord.end(); + // 结果集元数据 + ResultSetMetaData metaData = rs.getMetaData(); + columnNumber = metaData.getColumnCount(); + // 这个统计干净的result_Next时间 + PerfRecord allResultPerfRecord = new PerfRecord(taskGroupId, taskId, PerfRecord.PHASE.RESULT_NEXT_ALL); + allResultPerfRecord.start(); + // 传输数据 + long rsNextUsedTime = 0; + long lastTime = System.nanoTime(); + while (rs.next()) { + rsNextUsedTime += (System.nanoTime() - lastTime); + // 传输一条数据(记录) + this.transportOneRecord(readerSliceConfig, recordSender, rs, metaData, columnNumber, mandatoryEncoding, + taskPluginCollector); + lastTime = System.nanoTime(); + } + allResultPerfRecord.end(rsNextUsedTime); + // 目前大盘是依赖这个打印，而之前这个Finish read record是包含了sql查询和result next的全部时间 + LOG.info("Finished read record by Sql: [{}\n] {}.", querySql, basicMsg); + + } catch (Exception e) { + throw RdbmsException.asQueryException(this.dataBaseType, e, querySql, table, username); + } finally { + DBUtil.closeDBResources(null, conn); + } + } + + public void post(Configuration originalConfig) { + // do nothing + } + + public void destroy(Configuration originalConfig) { + // do nothing + } + + // 传输一个记录 + protected Record transportOneRecord(Configuration readerSliceConfig, RecordSender recordSender, ResultSet rs, ResultSetMetaData metaData, + int columnNumber, String mandatoryEncoding, TaskPluginCollector taskPluginCollector) { + Record record = buildRecord(readerSliceConfig, recordSender, rs, metaData, columnNumber, mandatoryEncoding, + taskPluginCollector); + recordSender.sendToWriter(record); + return record; + } + + // 构建记录，resultset -> record + // TableRecord RelRecord + + protected Record buildRecord(Configuration readerSliceConfig, RecordSender recordSender, ResultSet rs, ResultSetMetaData metaData, + int columnNumber, String mandatoryEncoding, TaskPluginCollector taskPluginCollector) { + Record record = phase.createRecord(readerSliceConfig); + + try { + for (int i = 1; i <= columnNumber; i++) { + switch (metaData.getColumnType(i)) { + + case Types.CHAR: + case Types.NCHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + case Types.NVARCHAR: + case Types.LONGNVARCHAR: + String rawData; + if (StringUtils.isBlank(mandatoryEncoding)) { + rawData = rs.getString(i); + } else { + rawData = new String((rs.getBytes(i) == null ? EMPTY_CHAR_ARRAY : rs.getBytes(i)), + mandatoryEncoding); + } + record.addColumn(new StringColumn(rawData)); + break; + + case Types.CLOB: + case Types.NCLOB: + record.addColumn(new StringColumn(rs.getString(i))); + break; + + case Types.SMALLINT: + case Types.TINYINT: + case Types.INTEGER: + case Types.BIGINT: + record.addColumn(new LongColumn(rs.getString(i))); + break; + + case Types.NUMERIC: + case Types.DECIMAL: + record.addColumn(new DoubleColumn(rs.getString(i))); + break; + + case Types.FLOAT: + case Types.REAL: + case Types.DOUBLE: + record.addColumn(new DoubleColumn(rs.getString(i))); + break; + + case Types.TIME: + record.addColumn(new DateColumn(rs.getTime(i))); + break; + + // for mysql bug, see http://bugs.mysql.com/bug.php?id=35115 + case Types.DATE: + if (metaData.getColumnTypeName(i).equalsIgnoreCase("year")) { + record.addColumn(new LongColumn(rs.getInt(i))); + } else { + record.addColumn(new DateColumn(rs.getDate(i))); + } + break; + + case Types.TIMESTAMP: + record.addColumn(new DateColumn(rs.getTimestamp(i))); + break; + + case Types.BINARY: + case Types.VARBINARY: + case Types.BLOB: + case Types.LONGVARBINARY: + record.addColumn(new BytesColumn(rs.getBytes(i))); + break; + + // warn: bit(1) -> Types.BIT 可使用BoolColumn + // warn: bit(>1) -> Types.VARBINARY 可使用BytesColumn + case Types.BOOLEAN: + case Types.BIT: + record.addColumn(new BoolColumn(rs.getBoolean(i))); + break; + + case Types.NULL: + String stringData = null; + if (rs.getObject(i) != null) { + stringData = rs.getObject(i).toString(); + } + record.addColumn(new StringColumn(stringData)); + break; + + default: + throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, String.format( + "您的配置文件中的列配置信息有误. 因为DataX 不支持数据库读取这种字段类型. 字段名:[%s], 字段名称:[%s], 字段Java类型:[%s]. 请尝试使用数据库函数将其转换datax支持的类型 或者不同步该字段 .", + metaData.getColumnName(i), metaData.getColumnType(i), metaData.getColumnClassName(i))); + } + } + } catch (Exception e) { + if (IS_DEBUG) { + LOG.debug("read data " + record.toString() + " occur exception:", e); + } + // TODO 这里识别为脏数据靠谱吗？ + taskPluginCollector.collectDirtyRecord(record, e); + if (e instanceof DataXException) { + throw (DataXException) e; + } + } + return record; + } + } + +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/Constant.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/Constant.java new file mode 100644 index 00000000..9c000f69 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/Constant.java @@ -0,0 +1,38 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader; + +public final class Constant { + public static final String PK_TYPE = "pkType"; + + public static final Object PK_TYPE_STRING = "pkTypeString"; + + public static final Object PK_TYPE_LONG = "pkTypeLong"; + + public static final Object PK_TYPE_MONTECARLO = "pkTypeMonteCarlo"; + + public static final String SPLIT_MODE_RANDOMSAMPLE = "randomSampling"; + + public static String CONN_MARK = "connection"; + + public static String TABLE_NUMBER_MARK = "tableNumber"; + + public static String IS_TABLE_MODE = "isTableMode"; + + public final static String FETCH_SIZE = "fetchSize"; + + public static String QUERY_SQL_TEMPLATE_WITHOUT_WHERE = "select %s from %s "; + + public static String QUERY_SQL_TEMPLATE = "select %s from %s where (%s)"; + + public static String TABLE_NAME_PLACEHOLDER = "@table"; + + public static String RDB_SCHEMA_URI = "schemaUri"; + + /** + * 关系 + */ + // 起点表/外键 + public static String REL_FROM = "relFrom"; + public static String REL_FK = "relFk"; + + public static String REL_QUERY_SQL_PATTERN = "select {0}, {1} from {2} where {3}"; +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/Key.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/Key.java new file mode 100644 index 00000000..3f8d0ff6 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/Key.java @@ -0,0 +1,53 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader; + +/** + * 编码，时区等配置，暂未定. + */ +public final class Key { + public final static String JDBC_URL = "jdbcUrl"; + + public final static String USERNAME = "username"; + + public final static String PASSWORD = "password"; + + public final static String TABLE = "table"; + + public final static String MANDATORY_ENCODING = "mandatoryEncoding"; + + // 数组配置 + public final static String COLUMN = "column"; + + public final static String COLUMN_LIST = "columnList"; + + public final static String WHERE = "where"; + + public final static String HINT = "hint"; + + public final static String SPLIT_PK = "splitPk"; + + public final static String SPLIT_MODE = "splitMode"; + + public final static String SAMPLE_PERCENTAGE = "samplePercentage"; + + public final static String QUERY_SQL = "querySql"; + + public final static String SPLIT_PK_SQL = "splitPkSql"; + + + public final static String PRE_SQL = "preSql"; + + public final static String POST_SQL = "postSql"; + + public final static String CHECK_SLAVE = "checkSlave"; + + public final static String SESSION = "session"; + + public final static String DBNAME = "dbName"; + + public final static String DRYRUN = "dryRun"; + + // graph + public final static String GRAPH_PHASE = "phase"; + + +} \ No newline at end of file diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/Phase.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/Phase.java new file mode 100644 index 00000000..4a4cd8eb --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/Phase.java @@ -0,0 +1,44 @@ +/** + * %% + * %% + */ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader; + +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.util.Configuration; +import com.leehom.arch.datax.plugin.rdb2graph.common.RelRecord; +import com.leehom.arch.datax.plugin.rdb2graph.common.TableRecord; + +/** + * @类名: Phase + * @说明: 读入阶段 + * + * @author leehom + * @Date 2020年9月3日 下午6:01:12s + * 修改记录： + * + * @see + */ +public enum Phase { + + TABLE, // 读入表 + REL // 表关系 + ; + // + public Record createRecord(Configuration readerSliceConfig) { + if(this==Phase.TABLE) { + String table = readerSliceConfig.getString(Key.TABLE); + TableRecord tr = new TableRecord(); + tr.setTable(table); + return tr; + } + else { + String table = readerSliceConfig.getString(Constant.REL_FROM); + String fk = readerSliceConfig.getString(Constant.REL_FK); + RelRecord rr = new RelRecord(); + rr.setFromTable(table); + rr.setFk(fk); + return rr; + } + } +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/ResultSetReadProxy.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/ResultSetReadProxy.java new file mode 100644 index 00000000..0218d67f --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/ResultSetReadProxy.java @@ -0,0 +1,144 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Types; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.datax.common.element.BoolColumn; +import com.alibaba.datax.common.element.BytesColumn; +import com.alibaba.datax.common.element.DateColumn; +import com.alibaba.datax.common.element.DoubleColumn; +import com.alibaba.datax.common.element.LongColumn; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.element.StringColumn; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtilErrorCode; + +public class ResultSetReadProxy { + private static final Logger LOG = LoggerFactory.getLogger(ResultSetReadProxy.class); + + private static final boolean IS_DEBUG = LOG.isDebugEnabled(); + private static final byte[] EMPTY_CHAR_ARRAY = new byte[0]; + + //TODO + public static void transportOneRecord(RecordSender recordSender, ResultSet rs, + ResultSetMetaData metaData, int columnNumber, String mandatoryEncoding, + TaskPluginCollector taskPluginCollector) { + Record record = recordSender.createRecord(); + + try { + for (int i = 1; i <= columnNumber; i++) { + switch (metaData.getColumnType(i)) { + + case Types.CHAR: + case Types.NCHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + case Types.NVARCHAR: + case Types.LONGNVARCHAR: + String rawData; + if(StringUtils.isBlank(mandatoryEncoding)){ + rawData = rs.getString(i); + }else{ + rawData = new String((rs.getBytes(i) == null ? EMPTY_CHAR_ARRAY : + rs.getBytes(i)), mandatoryEncoding); + } + record.addColumn(new StringColumn(rawData)); + break; + + case Types.CLOB: + case Types.NCLOB: + record.addColumn(new StringColumn(rs.getString(i))); + break; + + case Types.SMALLINT: + case Types.TINYINT: + case Types.INTEGER: + case Types.BIGINT: + record.addColumn(new LongColumn(rs.getString(i))); + break; + + case Types.NUMERIC: + case Types.DECIMAL: + record.addColumn(new DoubleColumn(rs.getString(i))); + break; + + case Types.FLOAT: + case Types.REAL: + case Types.DOUBLE: + record.addColumn(new DoubleColumn(rs.getString(i))); + break; + + case Types.TIME: + record.addColumn(new DateColumn(rs.getTime(i))); + break; + + // for mysql bug, see http://bugs.mysql.com/bug.php?id=35115 + case Types.DATE: + if (metaData.getColumnTypeName(i).equalsIgnoreCase("year")) { + record.addColumn(new LongColumn(rs.getInt(i))); + } else { + record.addColumn(new DateColumn(rs.getDate(i))); + } + break; + + case Types.TIMESTAMP: + record.addColumn(new DateColumn(rs.getTimestamp(i))); + break; + + case Types.BINARY: + case Types.VARBINARY: + case Types.BLOB: + case Types.LONGVARBINARY: + record.addColumn(new BytesColumn(rs.getBytes(i))); + break; + + // warn: bit(1) -> Types.BIT 可使用BoolColumn + // warn: bit(>1) -> Types.VARBINARY 可使用BytesColumn + case Types.BOOLEAN: + case Types.BIT: + record.addColumn(new BoolColumn(rs.getBoolean(i))); + break; + + case Types.NULL: + String stringData = null; + if(rs.getObject(i) != null) { + stringData = rs.getObject(i).toString(); + } + record.addColumn(new StringColumn(stringData)); + break; + + // TODO 添加BASIC_MESSAGE + default: + throw DataXException + .asDataXException( + DBUtilErrorCode.UNSUPPORTED_TYPE, + String.format( + "您的配置文件中的列配置信息有误. 因为DataX 不支持数据库读取这种字段类型. 字段名:[%s], 字段名称:[%s], 字段Java类型:[%s]. 请尝试使用数据库函数将其转换datax支持的类型 或者不同步该字段 .", + metaData.getColumnName(i), + metaData.getColumnType(i), + metaData.getColumnClassName(i))); + } + } + } catch (Exception e) { + if (IS_DEBUG) { + LOG.debug("read data " + record.toString() + + " occur exception:", e); + } + + //TODO 这里识别为脏数据靠谱吗？ + taskPluginCollector.collectDirtyRecord(record, e); + if (e instanceof DataXException) { + throw (DataXException) e; + } + } + + recordSender.sendToWriter(record); + } +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/HintUtil.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/HintUtil.java new file mode 100644 index 00000000..932e38b3 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/HintUtil.java @@ -0,0 +1,68 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.util; + +import com.alibaba.datax.common.util.Configuration; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Constant; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Key; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DataBaseType; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Created by liuyi on 15/9/18. + */ +public class HintUtil { + private static final Logger LOG = LoggerFactory.getLogger(ReaderSplitUtil.class); + + private static DataBaseType dataBaseType; + private static String username; + private static String password; + private static Pattern tablePattern; + private static String hintExpression; + + public static void initHintConf(DataBaseType type, Configuration configuration){ + dataBaseType = type; + username = configuration.getString(Key.USERNAME); + password = configuration.getString(Key.PASSWORD); + String hint = configuration.getString(Key.HINT); + if(StringUtils.isNotBlank(hint)){ + String[] tablePatternAndHint = hint.split("#"); + if(tablePatternAndHint.length==1){ + tablePattern = Pattern.compile(".*"); + hintExpression = tablePatternAndHint[0]; + }else{ + tablePattern = Pattern.compile(tablePatternAndHint[0]); + hintExpression = tablePatternAndHint[1]; + } + } + } + + public static String buildQueryColumn(String jdbcUrl, String table, String column){ + try{ + if(tablePattern != null && DataBaseType.Oracle.equals(dataBaseType)) { + Matcher m = tablePattern.matcher(table); + if(m.find()){ + String[] tableStr = table.split("\\."); + String tableWithoutSchema = tableStr[tableStr.length-1]; + String finalHint = hintExpression.replaceAll(Constant.TABLE_NAME_PLACEHOLDER, tableWithoutSchema); + //主库不并发读取 + if(finalHint.indexOf("parallel") > 0 && DBUtil.isOracleMaster(jdbcUrl, username, password)){ + LOG.info("master:{} will not use hint:{}", jdbcUrl, finalHint); + }else{ + LOG.info("table:{} use hint:{}.", table, finalHint); + return finalHint + column; + } + } + } + } catch (Exception e){ + LOG.warn("match hint exception, will not use hint", e); + } + return column; + } + +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/OriginalConfPretreatmentUtil.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/OriginalConfPretreatmentUtil.java new file mode 100644 index 00000000..1111e298 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/OriginalConfPretreatmentUtil.java @@ -0,0 +1,367 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.util; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.common.util.ListUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Constant; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Key; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Phase; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtilErrorCode; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DataBaseType; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.TableExpandUtil; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.SchemaUtils; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.DbSchema; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.TableMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.PKConstraintMetadata; +import com.leehom.arch.datax.plugin.rdb2graph.scanner.rdb.constraint.fk.FKConstraintMetadata; + +/** + * @类名: OriginalConfPretreatmentUtil + * @说明: 原生配置预处理 + * + * @author + * @Date 2022年4月29日 上午10:15:44 修改记录： + * + * @see + * + * TODO + * 1. 封装独立图配置预处理 + * + */ +public final class OriginalConfPretreatmentUtil { + private static final Logger LOG = LoggerFactory.getLogger(OriginalConfPretreatmentUtil.class); + + public static DataBaseType DATABASE_TYPE; + + /** + * @说明：原生配置预处理，这里增加我们关系转图配置 + * 1. table2node 表模式 + * 2. fk2rel querySql模式 + * + * @author leehom + * @param originalConfig + * + */ + public static void doPretreatment(Configuration originalConfig, DbSchema rdbSchema) { + // 检查 username/password 配置（必填） + originalConfig.getNecessaryValue(Key.USERNAME, DBUtilErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(Key.PASSWORD, DBUtilErrorCode.REQUIRED_VALUE); + // + graphconf(originalConfig, rdbSchema); + // + dealWhere(originalConfig); + // + simplifyConf(originalConfig); + } + + // + private static void graphconf(Configuration originalConfig, DbSchema rdbSchema) { + String phaseStr = originalConfig.getString(Key.GRAPH_PHASE); + Phase phase = Phase.valueOf(phaseStr.toUpperCase()); + if(phase==Phase.TABLE) + graphconfTable(originalConfig, rdbSchema); + if(phase==Phase.REL) + graphconfRel(originalConfig, rdbSchema); + } + + private static void graphconfTable(Configuration originalConfig, DbSchema rdbSchema) { + // 连接配置集合 + List conns = originalConfig.getList(Constant.CONN_MARK, Object.class); + // 遍历 + for (int i = 0, len = conns.size(); i < len; i++) { + // 获取表 + List tables = originalConfig.getList(String.format("%s[%d].%s", Constant.CONN_MARK, i, Key.TABLE), String.class); + // 设置了table不使用schema，这样支持表内分片 + if(tables.isEmpty()) { + // + String[] tns = SchemaUtils.extractTableNames("", rdbSchema.getTables()); + // + originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, i, Key.TABLE), tns); + } + } + } + + private static void graphconfRel(Configuration originalConfig, DbSchema rdbSchema) { + // 连接配置集合 + List conns = originalConfig.getList(Constant.CONN_MARK, Object.class); + List tables = rdbSchema.getTables(); + for (int i = 0, len = conns.size(); i < len; i++) { + // + List querySqls = new ArrayList<>(); + List relFroms = new ArrayList<>(); + List relFks = new ArrayList<>(); + // 按表遍历，但数量是按fk + for(int j=0;j fks = table.getFks(); + for(FKConstraintMetadata fk : fks) { + // 构建关系 querySql模式 + // 占位 + PKConstraintMetadata pk = table.getPk(); + // 0：主键，1：外键，2：表 + String relFromFields = SchemaUtils.extractFieldNames(pk.getFields()); + // fk字段不为空 + String whereSql = SchemaUtils.extractFieldWhereNotNull(fk.getFields(), " and "); + String relToFields = SchemaUtils.extractFieldNames(fk.getFields()); + String querySql = MessageFormat.format(Constant.REL_QUERY_SQL_PATTERN, + relFromFields, relToFields, table.getName(), whereSql); + querySqls.add(querySql); + relFroms.add(table.getName()); + relFks.add(fk.getFkName()); + } + } + } + // 写入connection节点 + originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, i, Key.QUERY_SQL), querySqls); + originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, i, Constant.REL_FROM), relFroms); + originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, i, Constant.REL_FK), relFks); + } + + } + + // 获取表邻接数组位置 + private static int locate(String tableName, List tables) { + int l = 0; + for(TableMetadata table : tables) { + if(tableName.equals(table.getName())) + break; + l++; + } + return l; + + } + + public static void dealWhere(Configuration originalConfig) { + String where = originalConfig.getString(Key.WHERE, null); + if (StringUtils.isNotBlank(where)) { + String whereImprove = where.trim(); + if (whereImprove.endsWith(";") || whereImprove.endsWith("；")) { + whereImprove = whereImprove.substring(0, whereImprove.length() - 1); + } + originalConfig.set(Key.WHERE, whereImprove); + } + } + + /** + * 对配置进行初步处理： + *
    + *
  1. 处理同一个数据库配置了多个jdbcUrl的情况
    2. + *
  2. 识别并标记是采用querySql 模式还是 table 模式
    3. + *
  3. 对 table 模式，确定分表个数，并处理 column 转 *事项
    4. + *
+ */ + private static void simplifyConf(Configuration originalConfig) { + boolean isTableMode = recognizeTableOrQuerySqlMode(originalConfig); + originalConfig.set(Constant.IS_TABLE_MODE, isTableMode); + // + dealJdbcAndTable(originalConfig); + dealColumnConf(originalConfig); + } + + private static void dealJdbcAndTable(Configuration originalConfig) { + String username = originalConfig.getString(Key.USERNAME); + String password = originalConfig.getString(Key.PASSWORD); + boolean checkSlave = originalConfig.getBool(Key.CHECK_SLAVE, false); + boolean isTableMode = originalConfig.getBool(Constant.IS_TABLE_MODE); + boolean isPreCheck = originalConfig.getBool(Key.DRYRUN, false); + + List conns = originalConfig.getList(Constant.CONN_MARK, Object.class); + List preSql = originalConfig.getList(Key.PRE_SQL, String.class); + + int tableNum = 0; + // 遍历连接配置 + for (int i = 0, len = conns.size(); i < len; i++) { + Configuration connConf = Configuration.from(conns.get(i).toString()); + + connConf.getNecessaryValue(Key.JDBC_URL, DBUtilErrorCode.REQUIRED_VALUE); + List jdbcUrls = connConf.getList(Key.JDBC_URL, String.class); + + String jdbcUrl; + if (isPreCheck) { + jdbcUrl = DBUtil.chooseJdbcUrlWithoutRetry(DATABASE_TYPE, jdbcUrls, username, password, preSql, + checkSlave); + } else { + jdbcUrl = DBUtil.chooseJdbcUrl(DATABASE_TYPE, jdbcUrls, username, password, preSql, checkSlave); + } + + jdbcUrl = DATABASE_TYPE.appendJDBCSuffixForReader(jdbcUrl); + + // 回写到connection[i].jdbcUrl + originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, i, Key.JDBC_URL), jdbcUrl); + + LOG.info("Available jdbcUrl:{}.", jdbcUrl); + + if (isTableMode) { + // table 方式 + // 对每一个connection 上配置的table 项进行解析(已对表名称进行了 ` 处理的) + List tables = connConf.getList(Key.TABLE, String.class); + // 支持分库分表 + List expandedTables = TableExpandUtil.expandTableConf(DATABASE_TYPE, tables); + if (null == expandedTables || expandedTables.isEmpty()) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE, + String.format("您所配置的读取数据库表:%s 不正确. 因为DataX根据您的配置找不到这张表. 请检查您的配置并作出修改." + "请先了解 DataX 配置.", + StringUtils.join(tables, ","))); + } + tableNum += expandedTables.size(); + + originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, i, Key.TABLE), expandedTables); + } else { + // 说明是配置的 querySql 方式，不做处理. + } + } + + originalConfig.set(Constant.TABLE_NUMBER_MARK, tableNum); + } + + // 列配置处理 + private static void dealColumnConf(Configuration originalConfig) { + boolean isTableMode = originalConfig.getBool(Constant.IS_TABLE_MODE); + + List userConfiguredColumns = originalConfig.getList(Key.COLUMN, String.class); + + if (isTableMode) { + if (null == userConfiguredColumns || userConfiguredColumns.isEmpty()) { + throw DataXException.asDataXException(DBUtilErrorCode.REQUIRED_VALUE, "您未配置读取数据库表的列信息. " + + "正确的配置方式是给 column 配置上您需要读取的列名称,用英文逗号分隔. 例如: \"column\": [\"id\", \"name\"],请参考上述配置并作出修改."); + } else { + String splitPk = originalConfig.getString(Key.SPLIT_PK, null); + + if (1 == userConfiguredColumns.size() && "*".equals(userConfiguredColumns.get(0))) { + LOG.warn("您的配置文件中的列配置存在一定的风险. 因为您未配置读取数据库表的列，当您的表字段个数、类型有变动时，可能影响任务正确性甚至会运行出错。请检查您的配置并作出修改."); + // 回填其值，需要以 String 的方式转交后续处理 + originalConfig.set(Key.COLUMN, "*"); + } else { + String jdbcUrl = originalConfig + .getString(String.format("%s[0].%s", Constant.CONN_MARK, Key.JDBC_URL)); + + String username = originalConfig.getString(Key.USERNAME); + String password = originalConfig.getString(Key.PASSWORD); + + String tableName = originalConfig + .getString(String.format("%s[0].%s[0]", Constant.CONN_MARK, Key.TABLE)); + + List allColumns = DBUtil.getTableColumns(DATABASE_TYPE, jdbcUrl, username, password, + tableName); + LOG.info("table:[{}] has columns:[{}].", tableName, StringUtils.join(allColumns, ",")); + // warn:注意mysql表名区分大小写 + allColumns = ListUtil.valueToLowerCase(allColumns); + List quotedColumns = new ArrayList(); + + for (String column : userConfiguredColumns) { + if ("*".equals(column)) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE, + "您的配置文件中的列配置信息有误. 因为根据您的配置，数据库表的列中存在多个*. 请检查您的配置并作出修改. "); + } + + quotedColumns.add(column); + // 以下判断没有任何意义 +// if (null == column) { +// quotedColumns.add(null); +// } else { +// if (allColumns.contains(column.toLowerCase())) { +// quotedColumns.add(column); +// } else { +// // 可能是由于用户填写为函数，或者自己对字段进行了`处理或者常量 +// quotedColumns.add(column); +// } +// } + } + + originalConfig.set(Key.COLUMN_LIST, quotedColumns); + originalConfig.set(Key.COLUMN, StringUtils.join(quotedColumns, ",")); + if (StringUtils.isNotBlank(splitPk)) { + if (!allColumns.contains(splitPk.toLowerCase())) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + String.format("您的配置文件中的列配置信息有误. 因为根据您的配置，您读取的数据库表:%s 中没有主键名为:%s. 请检查您的配置并作出修改.", + tableName, splitPk)); + } + } + + } + } + } else { + // querySql模式，不希望配制 column，那样是混淆不清晰的 + if (null != userConfiguredColumns && userConfiguredColumns.size() > 0) { + LOG.warn("您的配置有误. 由于您读取数据库表采用了querySql的方式, 所以您不需要再配置 column. 如果您不想看到这条提醒，请移除您源头表中配置中的 column."); + originalConfig.remove(Key.COLUMN); + } + + // querySql模式，不希望配制 where，那样是混淆不清晰的 + String where = originalConfig.getString(Key.WHERE, null); + if (StringUtils.isNotBlank(where)) { + LOG.warn("您的配置有误. 由于您读取数据库表采用了querySql的方式, 所以您不需要再配置 where. 如果您不想看到这条提醒，请移除您源头表中配置中的 where."); + originalConfig.remove(Key.WHERE); + } + + // querySql模式，不希望配制 splitPk，那样是混淆不清晰的 + String splitPk = originalConfig.getString(Key.SPLIT_PK, null); + if (StringUtils.isNotBlank(splitPk)) { + LOG.warn("您的配置有误. 由于您读取数据库表采用了querySql的方式, 所以您不需要再配置 splitPk. 如果您不想看到这条提醒，请移除您源头表中配置中的 splitPk."); + originalConfig.remove(Key.SPLIT_PK); + } + } + + } + + // 识别是表模式还是querySql模式 + // 验证 + private static boolean recognizeTableOrQuerySqlMode(Configuration originalConfig) { + List conns = originalConfig.getList(Constant.CONN_MARK, Object.class); + // + List tableModeFlags = new ArrayList(); + List querySqlModeFlags = new ArrayList(); + + String table = null; + String querySql = null; + + boolean isTableMode = false; + boolean isQuerySqlMode = false; + // + for (int i = 0, len = conns.size(); i < len; i++) { + Configuration connConf = Configuration.from(conns.get(i).toString()); + table = connConf.getString(Key.TABLE, null); + querySql = connConf.getString(Key.QUERY_SQL, null); + + isTableMode = StringUtils.isNotBlank(table); + tableModeFlags.add(isTableMode); + + isQuerySqlMode = StringUtils.isNotBlank(querySql); + querySqlModeFlags.add(isQuerySqlMode); + + if (false == isTableMode && false == isQuerySqlMode) { + // table 和 querySql 二者均未配制 + throw DataXException.asDataXException(DBUtilErrorCode.TABLE_QUERYSQL_MISSING, + "配置有误. 因为table和querySql应该配置并且只能配置一个. 请检查您的配置并作出修改."); + } else if (true == isTableMode && true == isQuerySqlMode) { + // table 和 querySql 二者均配置 + throw DataXException.asDataXException(DBUtilErrorCode.TABLE_QUERYSQL_MIXED, + "配置凌乱了. 因为datax不能同时既配置table又配置querySql.请检查您的配置并作出修改."); + } + } + // 混合配制 table 和 querySql + if (!ListUtil.checkIfValueSame(tableModeFlags) || !ListUtil.checkIfValueSame(tableModeFlags)) { + throw DataXException.asDataXException(DBUtilErrorCode.TABLE_QUERYSQL_MIXED, + "配置凌乱了. 不能同时既配置table又配置querySql. 请检查您的配置并作出修改."); + } + return tableModeFlags.get(0); + } + +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/PreCheckTask.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/PreCheckTask.java new file mode 100644 index 00000000..a5932c32 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/PreCheckTask.java @@ -0,0 +1,101 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.util; + +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.druid.sql.parser.ParserException; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Key; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DataBaseType; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.RdbmsException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.util.List; +import java.util.concurrent.Callable; + +/** + * Created by judy.lt on 2015/6/4. + */ +public class PreCheckTask implements Callable{ + private static final Logger LOG = LoggerFactory.getLogger(PreCheckTask.class); + private String userName; + private String password; + private String splitPkId; + private Configuration connection; + private DataBaseType dataBaseType; + + public PreCheckTask(String userName, + String password, + Configuration connection, + DataBaseType dataBaseType, + String splitPkId){ + this.connection = connection; + this.userName=userName; + this.password=password; + this.dataBaseType = dataBaseType; + this.splitPkId = splitPkId; + } + + @Override + public Boolean call() throws DataXException { + String jdbcUrl = this.connection.getString(Key.JDBC_URL); + List querySqls = this.connection.getList(Key.QUERY_SQL, Object.class); + List splitPkSqls = this.connection.getList(Key.SPLIT_PK_SQL, Object.class); + List tables = this.connection.getList(Key.TABLE,Object.class); + Connection conn = DBUtil.getConnectionWithoutRetry(this.dataBaseType, jdbcUrl, + this.userName, password); + int fetchSize = 1; + if(DataBaseType.MySql.equals(dataBaseType) || DataBaseType.DRDS.equals(dataBaseType)) { + fetchSize = Integer.MIN_VALUE; + } + try{ + for (int i=0;i doSplit( + Configuration originalSliceConfig, int adviceNumber) { + boolean isTableMode = originalSliceConfig.getBool(Constant.IS_TABLE_MODE).booleanValue(); + int eachTableShouldSplittedNumber = -1; + if (isTableMode) { + // adviceNumber这里是channel数量大小, 即datax并发task数量 + // eachTableShouldSplittedNumber是单表应该切分的份数, 向上取整可能和adviceNumber没有比例关系了已经 + eachTableShouldSplittedNumber = calculateEachTableShouldSplittedNumber( + adviceNumber, originalSliceConfig.getInt(Constant.TABLE_NUMBER_MARK)); + } + // sql元素 + String column = originalSliceConfig.getString(Key.COLUMN); + String where = originalSliceConfig.getString(Key.WHERE, null); + // 连接配置，table|querySql/jdbcUrl + List conns = originalSliceConfig.getList(Constant.CONN_MARK, Object.class); + // 分片配置 + List splittedConfigs = new ArrayList(); + for (int i = 0, len = conns.size(); i < len; i++) { + // 分片配置 + Configuration sliceConfig = originalSliceConfig.clone(); + // connection配置 + Configuration connConf = Configuration.from(conns.get(i).toString()); + // 简化配置的获取 + String jdbcUrl = connConf.getString(Key.JDBC_URL); + sliceConfig.set(Key.JDBC_URL, jdbcUrl); + // 抽取 jdbcUrl 中的 ip/port 进行资源使用的打标，以提供给 core 做有意义的 shuffle 操作 + sliceConfig.set(CommonConstant.LOAD_BALANCE_RESOURCE_MARK, DataBaseType.parseIpFromJdbcUrl(jdbcUrl)); + // + sliceConfig.remove(Constant.CONN_MARK); + // + Configuration tempSlice; + // table 方式 + if (isTableMode) { + // 已在之前进行了扩展和`处理，可以直接使用 + List tables = connConf.getList(Key.TABLE, String.class); + + Validate.isTrue(null != tables && !tables.isEmpty(), "您读取数据库表配置错误."); + // + String splitPk = originalSliceConfig.getString(Key.SPLIT_PK, null); + + //最终切分份数不一定等于 eachTableShouldSplittedNumber + boolean needSplitTable = eachTableShouldSplittedNumber > 1 && StringUtils.isNotBlank(splitPk); + if (needSplitTable) { + if (tables.size() == 1) { + //原来:如果是单表的，主键切分num=num*2+1 + // splitPk is null这类的情况的数据量本身就比真实数据量少很多, 和channel大小比率关系时，不建议考虑 + //eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * 2 + 1;// 不应该加1导致长尾 + + //考虑其他比率数字?(splitPk is null, 忽略此长尾) + eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * 5; + } + // 尝试对每个表，切分为eachTableShouldSplittedNumber 份 + for (String table : tables) { + tempSlice = sliceConfig.clone(); + tempSlice.set(Key.TABLE, table); + List splittedSlices = + SingleTableSplitUtil.splitSingleTable(tempSlice, eachTableShouldSplittedNumber); + splittedConfigs.addAll(splittedSlices); + } + } else { + for (String table : tables) { + tempSlice = sliceConfig.clone(); + tempSlice.set(Key.TABLE, table); + String queryColumn = HintUtil.buildQueryColumn(jdbcUrl, table, column); + tempSlice.set(Key.QUERY_SQL, SingleTableSplitUtil.buildQuerySql(queryColumn, table, where)); + splittedConfigs.add(tempSlice); + } + } + } else { // querySql 方式 + List querySqls = connConf.getList(Key.QUERY_SQL, String.class); + List relFroms = connConf.getList(Constant.REL_FROM, String.class); + List relFks = connConf.getList(Constant.REL_FK, String.class); + + // TODO 是否check 配置为多条语句？？ + for (int j=0;j conns = queryConfig.getList(Constant.CONN_MARK, Object.class); + + for (int i = 0, len = conns.size(); i < len; i++){ + Configuration connConf = Configuration.from(conns.get(i).toString()); + List querys = new ArrayList(); + List splitPkQuerys = new ArrayList(); + String connPath = String.format("connection[%d]",i); + // 说明是配置的 table 方式 + if (isTableMode) { + // 已在之前进行了扩展和`处理，可以直接使用 + List tables = connConf.getList(Key.TABLE, String.class); + Validate.isTrue(null != tables && !tables.isEmpty(), "您读取数据库表配置错误."); + for (String table : tables) { + querys.add(SingleTableSplitUtil.buildQuerySql(column,table,where)); + if (splitPK != null && !splitPK.isEmpty()){ + splitPkQuerys.add(SingleTableSplitUtil.genPKSql(splitPK.trim(),table,where)); + } + } + if (!splitPkQuerys.isEmpty()){ + connConf.set(Key.SPLIT_PK_SQL,splitPkQuerys); + } + connConf.set(Key.QUERY_SQL,querys); + queryConfig.set(connPath,connConf); + } else { + // 说明是配置的 querySql 方式 + List sqls = connConf.getList(Key.QUERY_SQL, + String.class); + for (String querySql : sqls) { + querys.add(querySql); + } + connConf.set(Key.QUERY_SQL,querys); + queryConfig.set(connPath,connConf); + } + } + return queryConfig; + } + + private static int calculateEachTableShouldSplittedNumber(int adviceNumber, int tableNumber) { + double tempNum = 1.0 * adviceNumber / tableNumber; + return (int) Math.ceil(tempNum); + } + +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/SingleTableSplitUtil.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/SingleTableSplitUtil.java new file mode 100644 index 00000000..a356e2b9 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/reader/util/SingleTableSplitUtil.java @@ -0,0 +1,363 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.util; + +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.fastjson.JSON; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Constant; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Key; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.*; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigInteger; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; + +/** + * @类名: SingleTableSplitUtil + * @说明: 单表分片 + * + * @author + * @Date 2022年4月29日 上午10:03:05 + * 修改记录： + * + * @see + */ +public class SingleTableSplitUtil { + private static final Logger LOG = LoggerFactory.getLogger(SingleTableSplitUtil.class); + + public static DataBaseType DATABASE_TYPE; + + private SingleTableSplitUtil() { + } + + public static List splitSingleTable(Configuration configuration, int adviceNum) { + List pluginParams = new ArrayList(); + List rangeList; + String splitPkName = configuration.getString(Key.SPLIT_PK); + String column = configuration.getString(Key.COLUMN); + String table = configuration.getString(Key.TABLE); + String where = configuration.getString(Key.WHERE, null); + boolean hasWhere = StringUtils.isNotBlank(where); + + // String splitMode = configuration.getString(Key.SPLIT_MODE, ""); + // if (Constant.SPLIT_MODE_RANDOMSAMPLE.equals(splitMode) && DATABASE_TYPE == + // DataBaseType.Oracle) { + if (DATABASE_TYPE == DataBaseType.Oracle) { + rangeList = genSplitSqlForOracle(splitPkName, table, where, configuration, adviceNum); + // warn: mysql etc to be added... + } else { + Pair minMaxPK = getPkRange(configuration); + if (null == minMaxPK) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + "根据切分主键切分表失败. DataX 仅支持切分主键为一个,并且类型为整数或者字符串类型. 请尝试使用其他的切分主键或者联系 DBA 进行处理."); + } + + configuration.set(Key.QUERY_SQL, buildQuerySql(column, table, where)); + if (null == minMaxPK.getLeft() || null == minMaxPK.getRight()) { + // 切分后获取到的start/end 有 Null 的情况 + pluginParams.add(configuration); + return pluginParams; + } + + boolean isStringType = Constant.PK_TYPE_STRING.equals(configuration.getString(Constant.PK_TYPE)); + boolean isLongType = Constant.PK_TYPE_LONG.equals(configuration.getString(Constant.PK_TYPE)); + + if (isStringType) { + rangeList = RdbmsRangeSplitWrap.splitAndWrap(String.valueOf(minMaxPK.getLeft()), + String.valueOf(minMaxPK.getRight()), adviceNum, splitPkName, "'", DATABASE_TYPE); + } else if (isLongType) { + rangeList = RdbmsRangeSplitWrap.splitAndWrap(new BigInteger(minMaxPK.getLeft().toString()), + new BigInteger(minMaxPK.getRight().toString()), adviceNum, splitPkName); + } else { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + "您配置的切分主键(splitPk) 类型 DataX 不支持. DataX 仅支持切分主键为一个,并且类型为整数或者字符串类型. 请尝试使用其他的切分主键或者联系 DBA 进行处理."); + } + } + String tempQuerySql; + List allQuerySql = new ArrayList(); + + if (null != rangeList && !rangeList.isEmpty()) { + for (String range : rangeList) { + Configuration tempConfig = configuration.clone(); + + tempQuerySql = buildQuerySql(column, table, where) + (hasWhere ? " and " : " where ") + range; + + allQuerySql.add(tempQuerySql); + tempConfig.set(Key.QUERY_SQL, tempQuerySql); + pluginParams.add(tempConfig); + } + } else { + // pluginParams.add(configuration); // this is wrong for new & old split + Configuration tempConfig = configuration.clone(); + tempQuerySql = buildQuerySql(column, table, where) + (hasWhere ? " and " : " where ") + + String.format(" %s IS NOT NULL", splitPkName); + allQuerySql.add(tempQuerySql); + tempConfig.set(Key.QUERY_SQL, tempQuerySql); + pluginParams.add(tempConfig); + } + + // deal pk is null + Configuration tempConfig = configuration.clone(); + tempQuerySql = buildQuerySql(column, table, where) + (hasWhere ? " and " : " where ") + + String.format(" %s IS NULL", splitPkName); + + allQuerySql.add(tempQuerySql); + + LOG.info("After split(), allQuerySql=[\n{}\n].", StringUtils.join(allQuerySql, "\n")); + + tempConfig.set(Key.QUERY_SQL, tempQuerySql); + pluginParams.add(tempConfig); + + return pluginParams; + } + + public static String buildQuerySql(String column, String table, String where) { + String querySql; + + if (StringUtils.isBlank(where)) { + querySql = String.format(Constant.QUERY_SQL_TEMPLATE_WITHOUT_WHERE, column, table); + } else { + querySql = String.format(Constant.QUERY_SQL_TEMPLATE, column, table, where); + } + + return querySql; + } + + @SuppressWarnings("resource") + private static Pair getPkRange(Configuration configuration) { + String pkRangeSQL = genPKRangeSQL(configuration); + + int fetchSize = configuration.getInt(Constant.FETCH_SIZE); + String jdbcURL = configuration.getString(Key.JDBC_URL); + String username = configuration.getString(Key.USERNAME); + String password = configuration.getString(Key.PASSWORD); + String table = configuration.getString(Key.TABLE); + + Connection conn = DBUtil.getConnection(DATABASE_TYPE, jdbcURL, username, password); + Pair minMaxPK = checkSplitPk(conn, pkRangeSQL, fetchSize, table, username, configuration); + DBUtil.closeDBResources(null, null, conn); + return minMaxPK; + } + + public static void precheckSplitPk(Connection conn, String pkRangeSQL, int fetchSize, String table, + String username) { + Pair minMaxPK = checkSplitPk(conn, pkRangeSQL, fetchSize, table, username, null); + if (null == minMaxPK) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + "根据切分主键切分表失败. DataX 仅支持切分主键为一个,并且类型为整数或者字符串类型. 请尝试使用其他的切分主键或者联系 DBA 进行处理."); + } + } + + /** + * 检测splitPk的配置是否正确。 configuration为null, + * 是precheck的逻辑，不需要回写PK_TYPE到configuration中 + * + */ + private static Pair checkSplitPk(Connection conn, String pkRangeSQL, int fetchSize, String table, + String username, Configuration configuration) { + LOG.info("split pk [sql={}] is running... ", pkRangeSQL); + ResultSet rs = null; + Pair minMaxPK = null; + try { + try { + rs = DBUtil.query(conn, pkRangeSQL, fetchSize); + } catch (Exception e) { + throw RdbmsException.asQueryException(DATABASE_TYPE, e, pkRangeSQL, table, username); + } + ResultSetMetaData rsMetaData = rs.getMetaData(); + if (isPKTypeValid(rsMetaData)) { + if (isStringType(rsMetaData.getColumnType(1))) { + if (configuration != null) { + configuration.set(Constant.PK_TYPE, Constant.PK_TYPE_STRING); + } + while (DBUtil.asyncResultSetNext(rs)) { + minMaxPK = new ImmutablePair(rs.getString(1), rs.getString(2)); + } + } else if (isLongType(rsMetaData.getColumnType(1))) { + if (configuration != null) { + configuration.set(Constant.PK_TYPE, Constant.PK_TYPE_LONG); + } + + while (DBUtil.asyncResultSetNext(rs)) { + minMaxPK = new ImmutablePair(rs.getString(1), rs.getString(2)); + + // check: string shouldn't contain '.', for oracle + String minMax = rs.getString(1) + rs.getString(2); + if (StringUtils.contains(minMax, '.')) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + "您配置的DataX切分主键(splitPk)有误. 因为您配置的切分主键(splitPk) 类型 DataX 不支持. DataX 仅支持切分主键为一个,并且类型为整数或者字符串类型. 请尝试使用其他的切分主键或者联系 DBA 进行处理."); + } + } + } else { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + "您配置的DataX切分主键(splitPk)有误. 因为您配置的切分主键(splitPk) 类型 DataX 不支持. DataX 仅支持切分主键为一个,并且类型为整数或者字符串类型. 请尝试使用其他的切分主键或者联系 DBA 进行处理."); + } + } else { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + "您配置的DataX切分主键(splitPk)有误. 因为您配置的切分主键(splitPk) 类型 DataX 不支持. DataX 仅支持切分主键为一个,并且类型为整数或者字符串类型. 请尝试使用其他的切分主键或者联系 DBA 进行处理."); + } + } catch (DataXException e) { + throw e; + } catch (Exception e) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, "DataX尝试切分表发生错误. 请检查您的配置并作出修改.", e); + } finally { + DBUtil.closeDBResources(rs, null, null); + } + + return minMaxPK; + } + + private static boolean isPKTypeValid(ResultSetMetaData rsMetaData) { + boolean ret = false; + try { + int minType = rsMetaData.getColumnType(1); + int maxType = rsMetaData.getColumnType(2); + + boolean isNumberType = isLongType(minType); + + boolean isStringType = isStringType(minType); + + if (minType == maxType && (isNumberType || isStringType)) { + ret = true; + } + } catch (Exception e) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + "DataX获取切分主键(splitPk)字段类型失败. 该错误通常是系统底层异常导致. 请联系旺旺:askdatax或者DBA处理."); + } + return ret; + } + + // warn: Types.NUMERIC is used for oracle! because oracle use NUMBER to + // store INT, SMALLINT, INTEGER etc, and only oracle need to concern + // Types.NUMERIC + private static boolean isLongType(int type) { + boolean isValidLongType = type == Types.BIGINT || type == Types.INTEGER || type == Types.SMALLINT + || type == Types.TINYINT; + + switch (SingleTableSplitUtil.DATABASE_TYPE) { + case Oracle: + isValidLongType |= type == Types.NUMERIC; + break; + default: + break; + } + return isValidLongType; + } + + private static boolean isStringType(int type) { + return type == Types.CHAR || type == Types.NCHAR || type == Types.VARCHAR || type == Types.LONGVARCHAR + || type == Types.NVARCHAR; + } + + private static String genPKRangeSQL(Configuration configuration) { + + String splitPK = configuration.getString(Key.SPLIT_PK).trim(); + String table = configuration.getString(Key.TABLE).trim(); + String where = configuration.getString(Key.WHERE, null); + return genPKSql(splitPK, table, where); + } + + public static String genPKSql(String splitPK, String table, String where) { + + String minMaxTemplate = "SELECT MIN(%s),MAX(%s) FROM %s"; + String pkRangeSQL = String.format(minMaxTemplate, splitPK, splitPK, table); + if (StringUtils.isNotBlank(where)) { + pkRangeSQL = String.format("%s WHERE (%s AND %s IS NOT NULL)", pkRangeSQL, where, splitPK); + } + return pkRangeSQL; + } + + /** + * support Number and String split + */ + public static List genSplitSqlForOracle(String splitPK, String table, String where, + Configuration configuration, int adviceNum) { + if (adviceNum < 1) { + throw new IllegalArgumentException(String.format("切分份数不能小于1. 此处:adviceNum=[%s].", adviceNum)); + } else if (adviceNum == 1) { + return null; + } + String whereSql = String.format("%s IS NOT NULL", splitPK); + if (StringUtils.isNotBlank(where)) { + whereSql = String.format(" WHERE (%s) AND (%s) ", whereSql, where); + } else { + whereSql = String.format(" WHERE (%s) ", whereSql); + } + Double percentage = configuration.getDouble(Key.SAMPLE_PERCENTAGE, 0.1); + String sampleSqlTemplate = "SELECT * FROM ( SELECT %s FROM %s SAMPLE (%s) %s ORDER BY DBMS_RANDOM.VALUE) WHERE ROWNUM <= %s ORDER by %s ASC"; + String splitSql = String.format(sampleSqlTemplate, splitPK, table, percentage, whereSql, adviceNum, splitPK); + + int fetchSize = configuration.getInt(Constant.FETCH_SIZE, 32); + String jdbcURL = configuration.getString(Key.JDBC_URL); + String username = configuration.getString(Key.USERNAME); + String password = configuration.getString(Key.PASSWORD); + Connection conn = DBUtil.getConnection(DATABASE_TYPE, jdbcURL, username, password); + LOG.info("split pk [sql={}] is running... ", splitSql); + ResultSet rs = null; + List> splitedRange = new ArrayList>(); + try { + try { + rs = DBUtil.query(conn, splitSql, fetchSize); + } catch (Exception e) { + throw RdbmsException.asQueryException(DATABASE_TYPE, e, splitSql, table, username); + } + if (configuration != null) { + configuration.set(Constant.PK_TYPE, Constant.PK_TYPE_MONTECARLO); + } + ResultSetMetaData rsMetaData = rs.getMetaData(); + while (DBUtil.asyncResultSetNext(rs)) { + ImmutablePair eachPoint = new ImmutablePair(rs.getObject(1), + rsMetaData.getColumnType(1)); + splitedRange.add(eachPoint); + } + } catch (DataXException e) { + throw e; + } catch (Exception e) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, "DataX尝试切分表发生错误. 请检查您的配置并作出修改.", e); + } finally { + DBUtil.closeDBResources(rs, null, null); + } + LOG.debug(JSON.toJSONString(splitedRange)); + List rangeSql = new ArrayList(); + int splitedRangeSize = splitedRange.size(); + // warn: splitedRangeSize may be 0 or 1，切分规则为IS NULL以及 IS NOT NULL + // demo: Parameter rangeResult can not be null and its length can not <2. + // detail:rangeResult=[24999930]. + if (splitedRangeSize >= 2) { + // warn: oracle Number is long type here + if (isLongType(splitedRange.get(0).getRight())) { + BigInteger[] integerPoints = new BigInteger[splitedRange.size()]; + for (int i = 0; i < splitedRangeSize; i++) { + integerPoints[i] = new BigInteger(splitedRange.get(i).getLeft().toString()); + } + rangeSql.addAll(RdbmsRangeSplitWrap.wrapRange(integerPoints, splitPK)); + // its ok if splitedRangeSize is 1 + rangeSql.add(RdbmsRangeSplitWrap.wrapFirstLastPoint(integerPoints[0], + integerPoints[splitedRangeSize - 1], splitPK)); + } else if (isStringType(splitedRange.get(0).getRight())) { + // warn: treated as string type + String[] stringPoints = new String[splitedRange.size()]; + for (int i = 0; i < splitedRangeSize; i++) { + stringPoints[i] = new String(splitedRange.get(i).getLeft().toString()); + } + rangeSql.addAll(RdbmsRangeSplitWrap.wrapRange(stringPoints, splitPK, "'", DATABASE_TYPE)); + // its ok if splitedRangeSize is 1 + rangeSql.add(RdbmsRangeSplitWrap.wrapFirstLastPoint(stringPoints[0], stringPoints[splitedRangeSize - 1], + splitPK, "'", DATABASE_TYPE)); + } else { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_SPLIT_PK, + "您配置的DataX切分主键(splitPk)有误. 因为您配置的切分主键(splitPk) 类型 DataX 不支持. DataX 仅支持切分主键为一个,并且类型为整数或者字符串类型. 请尝试使用其他的切分主键或者联系 DBA 进行处理."); + } + } + return rangeSql; + } +} \ No newline at end of file diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/ConnectionFactory.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/ConnectionFactory.java new file mode 100644 index 00000000..0d69e07c --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/ConnectionFactory.java @@ -0,0 +1,16 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.util; + +import java.sql.Connection; + +/** + * Date: 15/3/16 下午2:17 + */ +public interface ConnectionFactory { + + public Connection getConnecttion(); + + public Connection getConnecttionWithoutRetry(); + + public String getConnectionInfo(); + +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/Constant.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/Constant.java new file mode 100644 index 00000000..5775c8b4 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/Constant.java @@ -0,0 +1,27 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.util; + +public final class Constant { + static final int TIMEOUT_SECONDS = 15; + static final int MAX_TRY_TIMES = 4; + static final int SOCKET_TIMEOUT_INSECOND = 172800; + + public static final String MYSQL_DATABASE = "Unknown database"; + public static final String MYSQL_CONNEXP = "Communications link failure"; + public static final String MYSQL_ACCDENIED = "Access denied"; + public static final String MYSQL_TABLE_NAME_ERR1 = "Table"; + public static final String MYSQL_TABLE_NAME_ERR2 = "doesn't exist"; + public static final String MYSQL_SELECT_PRI = "SELECT command denied to user"; + public static final String MYSQL_COLUMN1 = "Unknown column"; + public static final String MYSQL_COLUMN2 = "field list"; + public static final String MYSQL_WHERE = "where clause"; + + public static final String ORACLE_DATABASE = "ORA-12505"; + public static final String ORACLE_CONNEXP = "The Network Adapter could not establish the connection"; + public static final String ORACLE_ACCDENIED = "ORA-01017"; + public static final String ORACLE_TABLE_NAME = "table or view does not exist"; + public static final String ORACLE_SELECT_PRI = "insufficient privileges"; + public static final String ORACLE_SQL = "invalid identifier"; + + + +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/DBUtil.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/DBUtil.java new file mode 100644 index 00000000..fab7ffce --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/DBUtil.java @@ -0,0 +1,803 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.util; + +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.common.util.RetryUtil; +import com.alibaba.druid.sql.parser.SQLParserUtils; +import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.reader.Key; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.apache.commons.lang3.tuple.Triple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.sql.*; +import java.util.*; +import java.util.concurrent.*; + +public final class DBUtil { + private static final Logger LOG = LoggerFactory.getLogger(DBUtil.class); + + private static final ThreadLocal rsExecutors = new ThreadLocal() { + @Override + protected ExecutorService initialValue() { + return Executors.newFixedThreadPool(1, new ThreadFactoryBuilder() + .setNameFormat("rsExecutors-%d") + .setDaemon(true) + .build()); + } + }; + + private DBUtil() { + } + + public static String chooseJdbcUrl(final DataBaseType dataBaseType, + final List jdbcUrls, final String username, + final String password, final List preSql, + final boolean checkSlave) { + + if (null == jdbcUrls || jdbcUrls.isEmpty()) { + throw DataXException.asDataXException( + DBUtilErrorCode.CONF_ERROR, + String.format("您的jdbcUrl的配置信息有错, 因为jdbcUrl[%s]不能为空. 请检查您的配置并作出修改.", + StringUtils.join(jdbcUrls, ","))); + } + + try { + return RetryUtil.executeWithRetry(new Callable() { + + @Override + public String call() throws Exception { + boolean connOK = false; + for (String url : jdbcUrls) { + if (StringUtils.isNotBlank(url)) { + url = url.trim(); + if (null != preSql && !preSql.isEmpty()) { + connOK = testConnWithoutRetry(dataBaseType, + url, username, password, preSql); + } else { + connOK = testConnWithoutRetry(dataBaseType, + url, username, password, checkSlave); + } + if (connOK) { + return url; + } + } + } + throw new Exception("DataX无法连接对应的数据库，可能原因是：1) 配置的ip/port/database/jdbc错误，无法连接。2) 配置的username/password错误，鉴权失败。请和DBA确认该数据库的连接信息是否正确。"); +// throw new Exception(DBUtilErrorCode.JDBC_NULL.toString()); + } + }, 7, 1000L, true); + //warn: 7 means 2 minutes + } catch (Exception e) { + throw DataXException.asDataXException( + DBUtilErrorCode.CONN_DB_ERROR, + String.format("数据库连接失败. 因为根据您配置的连接信息,无法从:%s 中找到可连接的jdbcUrl. 请检查您的配置并作出修改.", + StringUtils.join(jdbcUrls, ",")), e); + } + } + + public static String chooseJdbcUrlWithoutRetry(final DataBaseType dataBaseType, + final List jdbcUrls, final String username, + final String password, final List preSql, + final boolean checkSlave) throws DataXException { + + if (null == jdbcUrls || jdbcUrls.isEmpty()) { + throw DataXException.asDataXException( + DBUtilErrorCode.CONF_ERROR, + String.format("您的jdbcUrl的配置信息有错, 因为jdbcUrl[%s]不能为空. 请检查您的配置并作出修改.", + StringUtils.join(jdbcUrls, ","))); + } + + boolean connOK = false; + for (String url : jdbcUrls) { + if (StringUtils.isNotBlank(url)) { + url = url.trim(); + if (null != preSql && !preSql.isEmpty()) { + connOK = testConnWithoutRetry(dataBaseType, + url, username, password, preSql); + } else { + try { + connOK = testConnWithoutRetry(dataBaseType, + url, username, password, checkSlave); + } catch (Exception e) { + throw DataXException.asDataXException( + DBUtilErrorCode.CONN_DB_ERROR, + String.format("数据库连接失败. 因为根据您配置的连接信息,无法从:%s 中找到可连接的jdbcUrl. 请检查您的配置并作出修改.", + StringUtils.join(jdbcUrls, ",")), e); + } + } + if (connOK) { + return url; + } + } + } + throw DataXException.asDataXException( + DBUtilErrorCode.CONN_DB_ERROR, + String.format("数据库连接失败. 因为根据您配置的连接信息,无法从:%s 中找到可连接的jdbcUrl. 请检查您的配置并作出修改.", + StringUtils.join(jdbcUrls, ","))); + } + + /** + * 检查slave的库中的数据是否已到凌晨00:00 + * 如果slave同步的数据还未到00:00返回false + * 否则范围true + * + * @author ZiChi + * @version 1.0 2014-12-01 + */ + private static boolean isSlaveBehind(Connection conn) { + try { + ResultSet rs = query(conn, "SHOW VARIABLES LIKE 'read_only'"); + if (DBUtil.asyncResultSetNext(rs)) { + String readOnly = rs.getString("Value"); + if ("ON".equalsIgnoreCase(readOnly)) { //备库 + ResultSet rs1 = query(conn, "SHOW SLAVE STATUS"); + if (DBUtil.asyncResultSetNext(rs1)) { + String ioRunning = rs1.getString("Slave_IO_Running"); + String sqlRunning = rs1.getString("Slave_SQL_Running"); + long secondsBehindMaster = rs1.getLong("Seconds_Behind_Master"); + if ("Yes".equalsIgnoreCase(ioRunning) && "Yes".equalsIgnoreCase(sqlRunning)) { + ResultSet rs2 = query(conn, "SELECT TIMESTAMPDIFF(SECOND, CURDATE(), NOW())"); + DBUtil.asyncResultSetNext(rs2); + long secondsOfDay = rs2.getLong(1); + return secondsBehindMaster > secondsOfDay; + } else { + return true; + } + } else { + LOG.warn("SHOW SLAVE STATUS has no result"); + } + } + } else { + LOG.warn("SHOW VARIABLES like 'read_only' has no result"); + } + } catch (Exception e) { + LOG.warn("checkSlave failed, errorMessage:[{}].", e.getMessage()); + } + return false; + } + + /** + * 检查表是否具有insert 权限 + * insert on *.* 或者 insert on database.* 时验证通过 + * 当insert on database.tableName时，确保tableList中的所有table有insert 权限，验证通过 + * 其它验证都不通过 + * + * @author ZiChi + * @version 1.0 2015-01-28 + */ + public static boolean hasInsertPrivilege(DataBaseType dataBaseType, String jdbcURL, String userName, String password, List tableList) { + /*准备参数*/ + + String[] urls = jdbcURL.split("/"); + String dbName; + if (urls != null && urls.length != 0) { + dbName = urls[3]; + }else{ + return false; + } + + String dbPattern = "`" + dbName + "`.*"; + Collection tableNames = new HashSet(tableList.size()); + tableNames.addAll(tableList); + + Connection connection = connect(dataBaseType, jdbcURL, userName, password); + try { + ResultSet rs = query(connection, "SHOW GRANTS FOR " + userName); + while (DBUtil.asyncResultSetNext(rs)) { + String grantRecord = rs.getString("Grants for " + userName + "@%"); + String[] params = grantRecord.split("\\`"); + if (params != null && params.length >= 3) { + String tableName = params[3]; + if (params[0].contains("INSERT") && !tableName.equals("*") && tableNames.contains(tableName)) + tableNames.remove(tableName); + } else { + if (grantRecord.contains("INSERT") ||grantRecord.contains("ALL PRIVILEGES")) { + if (grantRecord.contains("*.*")) + return true; + else if (grantRecord.contains(dbPattern)) { + return true; + } + } + } + } + } catch (Exception e) { + LOG.warn("Check the database has the Insert Privilege failed, errorMessage:[{}]", e.getMessage()); + } + if (tableNames.isEmpty()) + return true; + return false; + } + + + public static boolean checkInsertPrivilege(DataBaseType dataBaseType, String jdbcURL, String userName, String password, List tableList) { + Connection connection = connect(dataBaseType, jdbcURL, userName, password); + String insertTemplate = "insert into %s(select * from %s where 1 = 2)"; + + boolean hasInsertPrivilege = true; + Statement insertStmt = null; + for(String tableName : tableList) { + String checkInsertPrivilegeSql = String.format(insertTemplate, tableName, tableName); + try { + insertStmt = connection.createStatement(); + executeSqlWithoutResultSet(insertStmt, checkInsertPrivilegeSql); + } catch (Exception e) { + if(DataBaseType.Oracle.equals(dataBaseType)) { + if(e.getMessage() != null && e.getMessage().contains("insufficient privileges")) { + hasInsertPrivilege = false; + LOG.warn("User [" + userName +"] has no 'insert' privilege on table[" + tableName + "], errorMessage:[{}]", e.getMessage()); + } + } else { + hasInsertPrivilege = false; + LOG.warn("User [" + userName + "] has no 'insert' privilege on table[" + tableName + "], errorMessage:[{}]", e.getMessage()); + } + } + } + try { + connection.close(); + } catch (SQLException e) { + LOG.warn("connection close failed, " + e.getMessage()); + } + return hasInsertPrivilege; + } + + public static boolean checkDeletePrivilege(DataBaseType dataBaseType,String jdbcURL, String userName, String password, List tableList) { + Connection connection = connect(dataBaseType, jdbcURL, userName, password); + String deleteTemplate = "delete from %s WHERE 1 = 2"; + + boolean hasInsertPrivilege = true; + Statement deleteStmt = null; + for(String tableName : tableList) { + String checkDeletePrivilegeSQL = String.format(deleteTemplate, tableName); + try { + deleteStmt = connection.createStatement(); + executeSqlWithoutResultSet(deleteStmt, checkDeletePrivilegeSQL); + } catch (Exception e) { + hasInsertPrivilege = false; + LOG.warn("User [" + userName +"] has no 'delete' privilege on table[" + tableName + "], errorMessage:[{}]", e.getMessage()); + } + } + try { + connection.close(); + } catch (SQLException e) { + LOG.warn("connection close failed, " + e.getMessage()); + } + return hasInsertPrivilege; + } + + public static boolean needCheckDeletePrivilege(Configuration originalConfig) { + List allSqls =new ArrayList(); + List preSQLs = originalConfig.getList(Key.PRE_SQL, String.class); + List postSQLs = originalConfig.getList(Key.POST_SQL, String.class); + if (preSQLs != null && !preSQLs.isEmpty()){ + allSqls.addAll(preSQLs); + } + if (postSQLs != null && !postSQLs.isEmpty()){ + allSqls.addAll(postSQLs); + } + for(String sql : allSqls) { + if(StringUtils.isNotBlank(sql)) { + if (sql.trim().toUpperCase().startsWith("DELETE")) { + return true; + } + } + } + return false; + } + + /** + * Get direct JDBC connection + *

+ * if connecting failed, try to connect for MAX_TRY_TIMES times + *

+ * NOTE: In DataX, we don't need connection pool in fact + */ + public static Connection getConnection(final DataBaseType dataBaseType, + final String jdbcUrl, final String username, final String password) { + + return getConnection(dataBaseType, jdbcUrl, username, password, String.valueOf(Constant.SOCKET_TIMEOUT_INSECOND * 1000)); + } + + /** + * + * @param dataBaseType + * @param jdbcUrl + * @param username + * @param password + * @param socketTimeout 设置socketTimeout，单位ms，String类型 + * @return + */ + public static Connection getConnection(final DataBaseType dataBaseType, + final String jdbcUrl, final String username, final String password, final String socketTimeout) { + + try { + return RetryUtil.executeWithRetry(new Callable() { + @Override + public Connection call() throws Exception { + return DBUtil.connect(dataBaseType, jdbcUrl, username, + password, socketTimeout); + } + }, 9, 1000L, true); + } catch (Exception e) { + throw DataXException.asDataXException( + DBUtilErrorCode.CONN_DB_ERROR, + String.format("数据库连接失败. 因为根据您配置的连接信息:%s获取数据库连接失败. 请检查您的配置并作出修改.", jdbcUrl), e); + } + } + + /** + * Get direct JDBC connection + *

+ * if connecting failed, try to connect for MAX_TRY_TIMES times + *

+ * NOTE: In DataX, we don't need connection pool in fact + */ + public static Connection getConnectionWithoutRetry(final DataBaseType dataBaseType, + final String jdbcUrl, final String username, final String password) { + return getConnectionWithoutRetry(dataBaseType, jdbcUrl, username, + password, String.valueOf(Constant.SOCKET_TIMEOUT_INSECOND * 1000)); + } + + public static Connection getConnectionWithoutRetry(final DataBaseType dataBaseType, + final String jdbcUrl, final String username, final String password, String socketTimeout) { + return DBUtil.connect(dataBaseType, jdbcUrl, username, + password, socketTimeout); + } + + private static synchronized Connection connect(DataBaseType dataBaseType, + String url, String user, String pass) { + return connect(dataBaseType, url, user, pass, String.valueOf(Constant.SOCKET_TIMEOUT_INSECOND * 1000)); + } + + private static synchronized Connection connect(DataBaseType dataBaseType, + String url, String user, String pass, String socketTimeout) { + + //ob10的处理 + if (url.startsWith(com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.Constant.OB10_SPLIT_STRING) && dataBaseType == DataBaseType.MySql) { + String[] ss = url.split(com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.Constant.OB10_SPLIT_STRING_PATTERN); + if (ss.length != 3) { + throw DataXException + .asDataXException( + DBUtilErrorCode.JDBC_OB10_ADDRESS_ERROR, "JDBC OB10格式错误，请联系askdatax"); + } + LOG.info("this is ob1_0 jdbc url."); + user = ss[1].trim() +":"+user; + url = ss[2]; + LOG.info("this is ob1_0 jdbc url. user="+user+" :url="+url); + } + + Properties prop = new Properties(); + prop.put("user", user); + prop.put("password", pass); + + if (dataBaseType == DataBaseType.Oracle) { + //oracle.net.READ_TIMEOUT for jdbc versions < 10.1.0.5 oracle.jdbc.ReadTimeout for jdbc versions >=10.1.0.5 + // unit ms + prop.put("oracle.jdbc.ReadTimeout", socketTimeout); + } + + return connect(dataBaseType, url, prop); + } + + private static synchronized Connection connect(DataBaseType dataBaseType, + String url, Properties prop) { + try { + Class.forName(dataBaseType.getDriverClassName()); + DriverManager.setLoginTimeout(Constant.TIMEOUT_SECONDS); + return DriverManager.getConnection(url, prop); + } catch (Exception e) { + throw RdbmsException.asConnException(dataBaseType, e, prop.getProperty("user"), null); + } + } + + /** + * a wrapped method to execute select-like sql statement . + * + * @param conn Database connection . + * @param sql sql statement to be executed + * @return a {@link ResultSet} + * @throws SQLException if occurs SQLException. + */ + public static ResultSet query(Connection conn, String sql, int fetchSize) + throws SQLException { + // 默认3600 s 的query Timeout + return query(conn, sql, fetchSize, Constant.SOCKET_TIMEOUT_INSECOND); + } + + /** + * a wrapped method to execute select-like sql statement . + * + * @param conn Database connection . + * @param sql sql statement to be executed + * @param fetchSize + * @param queryTimeout unit:second + * @return + * @throws SQLException + */ + public static ResultSet query(Connection conn, String sql, int fetchSize, int queryTimeout) + throws SQLException { + // make sure autocommit is off + conn.setAutoCommit(false); + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY); + stmt.setFetchSize(fetchSize); + stmt.setQueryTimeout(queryTimeout); + return query(stmt, sql); + } + + /** + * a wrapped method to execute select-like sql statement . + * + * @param stmt {@link Statement} + * @param sql sql statement to be executed + * @return a {@link ResultSet} + * @throws SQLException if occurs SQLException. + */ + public static ResultSet query(Statement stmt, String sql) + throws SQLException { + return stmt.executeQuery(sql); + } + + public static void executeSqlWithoutResultSet(Statement stmt, String sql) + throws SQLException { + stmt.execute(sql); + } + + /** + * Close {@link ResultSet}, {@link Statement} referenced by this + * {@link ResultSet} + * + * @param rs {@link ResultSet} to be closed + * @throws IllegalArgumentException + */ + public static void closeResultSet(ResultSet rs) { + try { + if (null != rs) { + Statement stmt = rs.getStatement(); + if (null != stmt) { + stmt.close(); + stmt = null; + } + rs.close(); + } + rs = null; + } catch (SQLException e) { + throw new IllegalStateException(e); + } + } + + public static void closeDBResources(ResultSet rs, Statement stmt, + Connection conn) { + if (null != rs) { + try { + rs.close(); + } catch (SQLException unused) { + } + } + + if (null != stmt) { + try { + stmt.close(); + } catch (SQLException unused) { + } + } + + if (null != conn) { + try { + conn.close(); + } catch (SQLException unused) { + } + } + } + + public static void closeDBResources(Statement stmt, Connection conn) { + closeDBResources(null, stmt, conn); + } + + public static List getTableColumns(DataBaseType dataBaseType, + String jdbcUrl, String user, String pass, String tableName) { + Connection conn = getConnection(dataBaseType, jdbcUrl, user, pass); + return getTableColumnsByConn(dataBaseType, conn, tableName, "jdbcUrl:"+jdbcUrl); + } + + public static List getTableColumnsByConn(DataBaseType dataBaseType, Connection conn, String tableName, String basicMsg) { + List columns = new ArrayList(); + Statement statement = null; + ResultSet rs = null; + String queryColumnSql = null; + try { + statement = conn.createStatement(); + queryColumnSql = String.format("select * from %s where 1=2", + tableName); + rs = statement.executeQuery(queryColumnSql); + ResultSetMetaData rsMetaData = rs.getMetaData(); + for (int i = 0, len = rsMetaData.getColumnCount(); i < len; i++) { + columns.add(rsMetaData.getColumnName(i + 1)); + } + + } catch (SQLException e) { + throw RdbmsException.asQueryException(dataBaseType,e,queryColumnSql,tableName,null); + } finally { + DBUtil.closeDBResources(rs, statement, conn); + } + + return columns; + } + + /** + * @return Left:ColumnName Middle:ColumnType Right:ColumnTypeName + */ + public static Triple, List, List> getColumnMetaData( + DataBaseType dataBaseType, String jdbcUrl, String user, + String pass, String tableName, String column) { + Connection conn = null; + try { + conn = getConnection(dataBaseType, jdbcUrl, user, pass); + return getColumnMetaData(conn, tableName, column); + } finally { + DBUtil.closeDBResources(null, null, conn); + } + } + + /** + * @return Left:ColumnName Middle:ColumnType Right:ColumnTypeName + */ + public static Triple, List, List> getColumnMetaData( + Connection conn, String tableName, String column) { + Statement statement = null; + ResultSet rs = null; + + Triple, List, List> columnMetaData = new ImmutableTriple, List, List>( + new ArrayList(), new ArrayList(), + new ArrayList()); + try { + statement = conn.createStatement(); + String queryColumnSql = "select " + column + " from " + tableName + + " where 1=2"; + + rs = statement.executeQuery(queryColumnSql); + ResultSetMetaData rsMetaData = rs.getMetaData(); + for (int i = 0, len = rsMetaData.getColumnCount(); i < len; i++) { + + columnMetaData.getLeft().add(rsMetaData.getColumnName(i + 1)); + columnMetaData.getMiddle().add(rsMetaData.getColumnType(i + 1)); + columnMetaData.getRight().add( + rsMetaData.getColumnTypeName(i + 1)); + } + return columnMetaData; + + } catch (SQLException e) { + throw DataXException + .asDataXException(DBUtilErrorCode.GET_COLUMN_INFO_FAILED, + String.format("获取表:%s 的字段的元信息时失败. 请联系 DBA 核查该库、表信息.", tableName), e); + } finally { + DBUtil.closeDBResources(rs, statement, null); + } + } + + public static boolean testConnWithoutRetry(DataBaseType dataBaseType, + String url, String user, String pass, boolean checkSlave){ + Connection connection = null; + + try { + connection = connect(dataBaseType, url, user, pass); + if (connection != null) { + if (dataBaseType.equals(dataBaseType.MySql) && checkSlave) { + //dataBaseType.MySql + boolean connOk = !isSlaveBehind(connection); + return connOk; + } else { + return true; + } + } + } catch (Exception e) { + LOG.warn("test connection of [{}] failed, for {}.", url, + e.getMessage()); + } finally { + DBUtil.closeDBResources(null, connection); + } + return false; + } + + public static boolean testConnWithoutRetry(DataBaseType dataBaseType, + String url, String user, String pass, List preSql) { + Connection connection = null; + try { + connection = connect(dataBaseType, url, user, pass); + if (null != connection) { + for (String pre : preSql) { + if (doPreCheck(connection, pre) == false) { + LOG.warn("doPreCheck failed."); + return false; + } + } + return true; + } + } catch (Exception e) { + LOG.warn("test connection of [{}] failed, for {}.", url, + e.getMessage()); + } finally { + DBUtil.closeDBResources(null, connection); + } + + return false; + } + + public static boolean isOracleMaster(final String url, final String user, final String pass) { + try { + return RetryUtil.executeWithRetry(new Callable() { + @Override + public Boolean call() throws Exception { + Connection conn = null; + try { + conn = connect(DataBaseType.Oracle, url, user, pass); + ResultSet rs = query(conn, "select DATABASE_ROLE from V$DATABASE"); + if (DBUtil.asyncResultSetNext(rs, 5)) { + String role = rs.getString("DATABASE_ROLE"); + return "PRIMARY".equalsIgnoreCase(role); + } + throw DataXException.asDataXException(DBUtilErrorCode.RS_ASYNC_ERROR, + String.format("select DATABASE_ROLE from V$DATABASE failed,请检查您的jdbcUrl:%s.", url)); + } finally { + DBUtil.closeDBResources(null, conn); + } + } + }, 3, 1000L, true); + } catch (Exception e) { + throw DataXException.asDataXException(DBUtilErrorCode.CONN_DB_ERROR, + String.format("select DATABASE_ROLE from V$DATABASE failed, url: %s", url), e); + } + } + + public static ResultSet query(Connection conn, String sql) + throws SQLException { + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY); + //默认3600 seconds + stmt.setQueryTimeout(Constant.SOCKET_TIMEOUT_INSECOND); + return query(stmt, sql); + } + + private static boolean doPreCheck(Connection conn, String pre) { + ResultSet rs = null; + try { + rs = query(conn, pre); + + int checkResult = -1; + if (DBUtil.asyncResultSetNext(rs)) { + checkResult = rs.getInt(1); + if (DBUtil.asyncResultSetNext(rs)) { + LOG.warn( + "pre check failed. It should return one result:0, pre:[{}].", + pre); + return false; + } + + } + + if (0 == checkResult) { + return true; + } + + LOG.warn( + "pre check failed. It should return one result:0, pre:[{}].", + pre); + } catch (Exception e) { + LOG.warn("pre check failed. pre:[{}], errorMessage:[{}].", pre, + e.getMessage()); + } finally { + DBUtil.closeResultSet(rs); + } + return false; + } + + // warn:until now, only oracle need to handle session config. + public static void dealWithSessionConfig(Connection conn, + Configuration config, DataBaseType databaseType, String message) { + List sessionConfig = null; + switch (databaseType) { + case Oracle: + sessionConfig = config.getList(Key.SESSION, + new ArrayList(), String.class); + DBUtil.doDealWithSessionConfig(conn, sessionConfig, message); + break; + case DRDS: + // 用于关闭 drds 的分布式事务开关 + sessionConfig = new ArrayList(); + sessionConfig.add("set transaction policy 4"); + DBUtil.doDealWithSessionConfig(conn, sessionConfig, message); + break; + case MySql: + sessionConfig = config.getList(Key.SESSION, + new ArrayList(), String.class); + DBUtil.doDealWithSessionConfig(conn, sessionConfig, message); + break; + default: + break; + } + } + + private static void doDealWithSessionConfig(Connection conn, + List sessions, String message) { + if (null == sessions || sessions.isEmpty()) { + return; + } + + Statement stmt; + try { + stmt = conn.createStatement(); + } catch (SQLException e) { + throw DataXException + .asDataXException(DBUtilErrorCode.SET_SESSION_ERROR, String + .format("session配置有误. 因为根据您的配置执行 session 设置失败. 上下文信息是:[%s]. 请检查您的配置并作出修改.", message), + e); + } + + for (String sessionSql : sessions) { + LOG.info("execute sql:[{}]", sessionSql); + try { + DBUtil.executeSqlWithoutResultSet(stmt, sessionSql); + } catch (SQLException e) { + throw DataXException.asDataXException( + DBUtilErrorCode.SET_SESSION_ERROR, String.format( + "session配置有误. 因为根据您的配置执行 session 设置失败. 上下文信息是:[%s]. 请检查您的配置并作出修改.", message), e); + } + } + DBUtil.closeDBResources(stmt, null); + } + + public static void sqlValid(String sql, DataBaseType dataBaseType){ + SQLStatementParser statementParser = SQLParserUtils.createSQLStatementParser(sql,dataBaseType.getTypeName()); + statementParser.parseStatementList(); + } + + /** + * 异步获取resultSet的next(),注意，千万不能应用在数据的读取中。只能用在meta的获取 + * @param resultSet + * @return + */ + public static boolean asyncResultSetNext(final ResultSet resultSet) { + return asyncResultSetNext(resultSet, 3600); + } + + public static boolean asyncResultSetNext(final ResultSet resultSet, int timeout) { + Future future = rsExecutors.get().submit(new Callable() { + @Override + public Boolean call() throws Exception { + return resultSet.next(); + } + }); + try { + return future.get(timeout, TimeUnit.SECONDS); + } catch (Exception e) { + throw DataXException.asDataXException( + DBUtilErrorCode.RS_ASYNC_ERROR, "异步获取ResultSet失败", e); + } + } + + public static void loadDriverClass(String pluginType, String pluginName) { + try { + String pluginJsonPath = StringUtils.join( + new String[] { System.getProperty("datax.home"), "plugin", + pluginType, + String.format("%s%s", pluginName, pluginType), + "plugin.json" }, File.separator); + Configuration configuration = Configuration.from(new File( + pluginJsonPath)); + List drivers = configuration.getList("drivers", + String.class); + for (String driver : drivers) { + Class.forName(driver); + } + } catch (ClassNotFoundException e) { + throw DataXException.asDataXException(DBUtilErrorCode.CONF_ERROR, + "数据库驱动加载错误, 请确认libs目录有驱动jar包且plugin.json中drivers配置驱动类正确!", + e); + } + } +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/DBUtilErrorCode.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/DBUtilErrorCode.java new file mode 100644 index 00000000..d3483d81 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/DBUtilErrorCode.java @@ -0,0 +1,99 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.util; + +import com.alibaba.datax.common.spi.ErrorCode; + +//TODO +public enum DBUtilErrorCode implements ErrorCode { + //连接错误 + MYSQL_CONN_USERPWD_ERROR("MYSQLErrCode-01","数据库用户名或者密码错误，请检查填写的账号密码或者联系DBA确认账号和密码是否正确"), + MYSQL_CONN_IPPORT_ERROR("MYSQLErrCode-02","数据库服务的IP地址或者Port错误，请检查填写的IP地址和Port或者联系DBA确认IP地址和Port是否正确。如果是同步中心用户请联系DBA确认idb上录入的IP和PORT信息和数据库的当前实际信息是一致的"), + MYSQL_CONN_DB_ERROR("MYSQLErrCode-03","数据库名称错误，请检查数据库实例名称或者联系DBA确认该实例是否存在并且在正常服务"), + + ORACLE_CONN_USERPWD_ERROR("ORACLEErrCode-01","数据库用户名或者密码错误，请检查填写的账号密码或者联系DBA确认账号和密码是否正确"), + ORACLE_CONN_IPPORT_ERROR("ORACLEErrCode-02","数据库服务的IP地址或者Port错误，请检查填写的IP地址和Port或者联系DBA确认IP地址和Port是否正确。如果是同步中心用户请联系DBA确认idb上录入的IP和PORT信息和数据库的当前实际信息是一致的"), + ORACLE_CONN_DB_ERROR("ORACLEErrCode-03","数据库名称错误，请检查数据库实例名称或者联系DBA确认该实例是否存在并且在正常服务"), + + //execute query错误 + MYSQL_QUERY_TABLE_NAME_ERROR("MYSQLErrCode-04","表不存在，请检查表名或者联系DBA确认该表是否存在"), + MYSQL_QUERY_SQL_ERROR("MYSQLErrCode-05","SQL语句执行出错，请检查Where条件是否存在拼写或语法错误"), + MYSQL_QUERY_COLUMN_ERROR("MYSQLErrCode-06","Column信息错误，请检查该列是否存在，如果是常量或者变量，请使用英文单引号’包起来"), + MYSQL_QUERY_SELECT_PRI_ERROR("MYSQLErrCode-07","读表数据出错，因为账号没有读表的权限，请联系DBA确认该账号的权限并授权"), + + ORACLE_QUERY_TABLE_NAME_ERROR("ORACLEErrCode-04","表不存在，请检查表名或者联系DBA确认该表是否存在"), + ORACLE_QUERY_SQL_ERROR("ORACLEErrCode-05","SQL语句执行出错，原因可能是你填写的列不存在或者where条件不符合要求，1，请检查该列是否存在，如果是常量或者变量，请使用英文单引号’包起来; 2，请检查Where条件是否存在拼写或语法错误"), + ORACLE_QUERY_SELECT_PRI_ERROR("ORACLEErrCode-06","读表数据出错，因为账号没有读表的权限，请联系DBA确认该账号的权限并授权"), + ORACLE_QUERY_SQL_PARSER_ERROR("ORACLEErrCode-07","SQL语法出错，请检查Where条件是否存在拼写或语法错误"), + + //PreSql,Post Sql错误 + MYSQL_PRE_SQL_ERROR("MYSQLErrCode-08","PreSQL语法错误，请检查"), + MYSQL_POST_SQL_ERROR("MYSQLErrCode-09","PostSql语法错误，请检查"), + MYSQL_QUERY_SQL_PARSER_ERROR("MYSQLErrCode-10","SQL语法出错，请检查Where条件是否存在拼写或语法错误"), + + ORACLE_PRE_SQL_ERROR("ORACLEErrCode-08", "PreSQL语法错误，请检查"), + ORACLE_POST_SQL_ERROR("ORACLEErrCode-09", "PostSql语法错误，请检查"), + + //SplitPK 错误 + MYSQL_SPLIT_PK_ERROR("MYSQLErrCode-11","SplitPK错误，请检查"), + ORACLE_SPLIT_PK_ERROR("ORACLEErrCode-10","SplitPK错误，请检查"), + + //Insert,Delete 权限错误 + MYSQL_INSERT_ERROR("MYSQLErrCode-12","数据库没有写权限，请联系DBA"), + MYSQL_DELETE_ERROR("MYSQLErrCode-13","数据库没有Delete权限，请联系DBA"), + ORACLE_INSERT_ERROR("ORACLEErrCode-11","数据库没有写权限，请联系DBA"), + ORACLE_DELETE_ERROR("ORACLEErrCode-12","数据库没有Delete权限，请联系DBA"), + + JDBC_NULL("DBUtilErrorCode-20","JDBC URL为空，请检查配置"), + JDBC_OB10_ADDRESS_ERROR("DBUtilErrorCode-OB10-01","JDBC OB10格式错误，请联系askdatax"), + CONF_ERROR("DBUtilErrorCode-00", "您的配置错误."), + CONN_DB_ERROR("DBUtilErrorCode-10", "连接数据库失败. 请检查您的 账号、密码、数据库名称、IP、Port或者向 DBA 寻求帮助(注意网络环境)."), + GET_COLUMN_INFO_FAILED("DBUtilErrorCode-01", "获取表字段相关信息失败."), + UNSUPPORTED_TYPE("DBUtilErrorCode-12", "不支持的数据库类型. 请注意查看 DataX 已经支持的数据库类型以及数据库版本."), + COLUMN_SPLIT_ERROR("DBUtilErrorCode-13", "根据主键进行切分失败."), + SET_SESSION_ERROR("DBUtilErrorCode-14", "设置 session 失败."), + RS_ASYNC_ERROR("DBUtilErrorCode-15", "异步获取ResultSet next失败."), + + REQUIRED_VALUE("DBUtilErrorCode-03", "您缺失了必须填写的参数值."), + ILLEGAL_VALUE("DBUtilErrorCode-02", "您填写的参数值不合法."), + ILLEGAL_SPLIT_PK("DBUtilErrorCode-04", "您填写的主键列不合法, DataX 仅支持切分主键为一个,并且类型为整数或者字符串类型."), + SPLIT_FAILED_ILLEGAL_SQL("DBUtilErrorCode-15", "DataX尝试切分表时, 执行数据库 Sql 失败. 请检查您的配置 table/splitPk/where 并作出修改."), + SQL_EXECUTE_FAIL("DBUtilErrorCode-06", "执行数据库 Sql 失败, 请检查您的配置的 column/table/where/querySql或者向 DBA 寻求帮助."), + + // only for reader + READ_RECORD_FAIL("DBUtilErrorCode-07", "读取数据库数据失败. 请检查您的配置的 column/table/where/querySql或者向 DBA 寻求帮助."), + TABLE_QUERYSQL_MIXED("DBUtilErrorCode-08", "获取数据模式配置错误. 不能同时既配置table又配置querySql"), + TABLE_QUERYSQL_MISSING("DBUtilErrorCode-09", "获取数据模式配置错误. table和querySql 应该并且只能配置一个."), + + // only for writer + WRITE_DATA_ERROR("DBUtilErrorCode-05", "往您配置的写入表中写入数据时失败."), + NO_INSERT_PRIVILEGE("DBUtilErrorCode-11", "数据库没有写权限，请联系DBA"), + NO_DELETE_PRIVILEGE("DBUtilErrorCode-16", "数据库没有DELETE权限，请联系DBA"), + + // graph + ERROR_LOAD_RDBSCHEMA("DBUtilErrorCode-17", "载入关系模式异常."), + ; + + private final String code; + + private final String description; + + private DBUtilErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s]. ", this.code, + this.description); + } +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/DataBaseType.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/DataBaseType.java new file mode 100644 index 00000000..dbee4dc2 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/DataBaseType.java @@ -0,0 +1,209 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.util; + +import com.alibaba.datax.common.exception.DataXException; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * refer:http://blog.csdn.net/ring0hx/article/details/6152528 + *

+ */ +public enum DataBaseType { + MySql("mysql", "com.mysql.jdbc.Driver", "select 'x'"), + Tddl("mysql", "com.mysql.jdbc.Driver", "select 'x'"), + DRDS("drds", "com.mysql.jdbc.Driver", "select 'x'"), + Oracle("oracle", "oracle.jdbc.OracleDriver", "select 'x' from dual"), + SQLServer("sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "select x"), + PostgreSQL("postgresql", "org.postgresql.Driver", "select x"), + RDBMS("rdbms", "com.alibaba.datax.plugin.rdbms.util.DataBaseType", "select x"), + DB2("db2", "com.ibm.db2.jcc.DB2Driver", "select x"), + ADS("ads","com.mysql.jdbc.Driver", "select x"), + ClickHouse("clickhouse", "ru.yandex.clickhouse.ClickHouseDriver", "select x"); + + + private String typeName; + private String driverClassName; + private String testSql; + + DataBaseType(String typeName, String driverClassName, String testSql) { + this.typeName = typeName; + this.driverClassName = driverClassName; + this.testSql = testSql; + } + + public String getDriverClassName() { + return this.driverClassName; + } + + public String appendJDBCSuffixForReader(String jdbc) { + String result = jdbc; + String suffix = null; + switch (this) { + case MySql: + case DRDS: + suffix = "yearIsDateType=false&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false&rewriteBatchedStatements=true"; + if (jdbc.contains("?")) { + result = jdbc + "&" + suffix; + } else { + result = jdbc + "?" + suffix; + } + break; + case Oracle: + break; + case SQLServer: + break; + case DB2: + break; + case PostgreSQL: + break; + case ClickHouse: + break; + case RDBMS: + break; + default: + throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type."); + } + + return result; + } + + public String appendJDBCSuffixForWriter(String jdbc) { + String result = jdbc; + String suffix = null; + switch (this) { + case MySql: + suffix = "yearIsDateType=false&zeroDateTimeBehavior=convertToNull&rewriteBatchedStatements=true&tinyInt1isBit=false"; + if (jdbc.contains("?")) { + result = jdbc + "&" + suffix; + } else { + result = jdbc + "?" + suffix; + } + break; + case DRDS: + suffix = "yearIsDateType=false&zeroDateTimeBehavior=convertToNull"; + if (jdbc.contains("?")) { + result = jdbc + "&" + suffix; + } else { + result = jdbc + "?" + suffix; + } + break; + case Oracle: + break; + case SQLServer: + break; + case DB2: + break; + case PostgreSQL: + break; + case ClickHouse: + break; + case RDBMS: + break; + default: + throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type."); + } + + return result; + } + + public String formatPk(String splitPk) { + String result = splitPk; + + switch (this) { + case MySql: + case Oracle: + if (splitPk.length() >= 2 && splitPk.startsWith("`") && splitPk.endsWith("`")) { + result = splitPk.substring(1, splitPk.length() - 1).toLowerCase(); + } + break; + case SQLServer: + if (splitPk.length() >= 2 && splitPk.startsWith("[") && splitPk.endsWith("]")) { + result = splitPk.substring(1, splitPk.length() - 1).toLowerCase(); + } + break; + case DB2: + case PostgreSQL: + break; + default: + throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type."); + } + + return result; + } + + + public String quoteColumnName(String columnName) { + String result = columnName; + + switch (this) { + case MySql: + result = "`" + columnName.replace("`", "``") + "`"; + break; + case Oracle: + break; + case SQLServer: + result = "[" + columnName + "]"; + break; + case DB2: + case PostgreSQL: + break; + default: + throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type"); + } + + return result; + } + + public String quoteTableName(String tableName) { + String result = tableName; + + switch (this) { + case MySql: + result = "`" + tableName.replace("`", "``") + "`"; + break; + case Oracle: + break; + case SQLServer: + break; + case DB2: + break; + case PostgreSQL: + break; + default: + throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type"); + } + + return result; + } + + private static Pattern mysqlPattern = Pattern.compile("jdbc:mysql://(.+):\\d+/.+"); + private static Pattern oraclePattern = Pattern.compile("jdbc:oracle:thin:@(.+):\\d+:.+"); + + /** + * 注意：目前只实现了从 mysql/oracle 中识别出ip 信息.未识别到则返回 null. + */ + public static String parseIpFromJdbcUrl(String jdbcUrl) { + Matcher mysql = mysqlPattern.matcher(jdbcUrl); + if (mysql.matches()) { + return mysql.group(1); + } + Matcher oracle = oraclePattern.matcher(jdbcUrl); + if (oracle.matches()) { + return oracle.group(1); + } + return null; + } + public String getTypeName() { + return typeName; + } + + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + public String getTestSql() { + return testSql; + } + +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/JdbcConnectionFactory.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/JdbcConnectionFactory.java new file mode 100644 index 00000000..57d3ba83 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/JdbcConnectionFactory.java @@ -0,0 +1,39 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.util; + +import java.sql.Connection; + +/** + * Date: 15/3/16 下午3:12 + */ +public class JdbcConnectionFactory implements ConnectionFactory { + + private DataBaseType dataBaseType; + + private String jdbcUrl; + + private String userName; + + private String password; + + public JdbcConnectionFactory(DataBaseType dataBaseType, String jdbcUrl, String userName, String password) { + this.dataBaseType = dataBaseType; + this.jdbcUrl = jdbcUrl; + this.userName = userName; + this.password = password; + } + + @Override + public Connection getConnecttion() { + return DBUtil.getConnection(dataBaseType, jdbcUrl, userName, password); + } + + @Override + public Connection getConnecttionWithoutRetry() { + return DBUtil.getConnectionWithoutRetry(dataBaseType, jdbcUrl, userName, password); + } + + @Override + public String getConnectionInfo() { + return "jdbcUrl:" + jdbcUrl; + } +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/RdbmsException.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/RdbmsException.java new file mode 100644 index 00000000..a03d04eb --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/RdbmsException.java @@ -0,0 +1,190 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.util; + +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.spi.ErrorCode; + +/** + * Created by judy.lt on 2015/6/5. + */ +public class RdbmsException extends DataXException{ + public RdbmsException(ErrorCode errorCode, String message){ + super(errorCode,message); + } + + public static DataXException asConnException(DataBaseType dataBaseType,Exception e,String userName,String dbName){ + if (dataBaseType.equals(DataBaseType.MySql)){ + DBUtilErrorCode dbUtilErrorCode = mySqlConnectionErrorAna(e.getMessage()); + if (dbUtilErrorCode == DBUtilErrorCode.MYSQL_CONN_DB_ERROR && dbName !=null ){ + return DataXException.asDataXException(dbUtilErrorCode,"该数据库名称为："+dbName+" 具体错误信息为："+e); + } + if (dbUtilErrorCode == DBUtilErrorCode.MYSQL_CONN_USERPWD_ERROR ){ + return DataXException.asDataXException(dbUtilErrorCode,"该数据库用户名为："+userName+" 具体错误信息为："+e); + } + return DataXException.asDataXException(dbUtilErrorCode," 具体错误信息为："+e); + } + + if (dataBaseType.equals(DataBaseType.Oracle)){ + DBUtilErrorCode dbUtilErrorCode = oracleConnectionErrorAna(e.getMessage()); + if (dbUtilErrorCode == DBUtilErrorCode.ORACLE_CONN_DB_ERROR && dbName != null){ + return DataXException.asDataXException(dbUtilErrorCode,"该数据库名称为："+dbName+" 具体错误信息为："+e); + } + if (dbUtilErrorCode == DBUtilErrorCode.ORACLE_CONN_USERPWD_ERROR ){ + return DataXException.asDataXException(dbUtilErrorCode,"该数据库用户名为："+userName+" 具体错误信息为："+e); + } + return DataXException.asDataXException(dbUtilErrorCode," 具体错误信息为："+e); + } + return DataXException.asDataXException(DBUtilErrorCode.CONN_DB_ERROR," 具体错误信息为："+e); + } + + public static DBUtilErrorCode mySqlConnectionErrorAna(String e){ + if (e.contains(Constant.MYSQL_DATABASE)){ + return DBUtilErrorCode.MYSQL_CONN_DB_ERROR; + } + + if (e.contains(Constant.MYSQL_CONNEXP)){ + return DBUtilErrorCode.MYSQL_CONN_IPPORT_ERROR; + } + + if (e.contains(Constant.MYSQL_ACCDENIED)){ + return DBUtilErrorCode.MYSQL_CONN_USERPWD_ERROR; + } + + return DBUtilErrorCode.CONN_DB_ERROR; + } + + public static DBUtilErrorCode oracleConnectionErrorAna(String e){ + if (e.contains(Constant.ORACLE_DATABASE)){ + return DBUtilErrorCode.ORACLE_CONN_DB_ERROR; + } + + if (e.contains(Constant.ORACLE_CONNEXP)){ + return DBUtilErrorCode.ORACLE_CONN_IPPORT_ERROR; + } + + if (e.contains(Constant.ORACLE_ACCDENIED)){ + return DBUtilErrorCode.ORACLE_CONN_USERPWD_ERROR; + } + + return DBUtilErrorCode.CONN_DB_ERROR; + } + + public static DataXException asQueryException(DataBaseType dataBaseType, Exception e,String querySql,String table,String userName){ + if (dataBaseType.equals(DataBaseType.MySql)){ + DBUtilErrorCode dbUtilErrorCode = mySqlQueryErrorAna(e.getMessage()); + if (dbUtilErrorCode == DBUtilErrorCode.MYSQL_QUERY_TABLE_NAME_ERROR && table != null){ + return DataXException.asDataXException(dbUtilErrorCode,"表名为："+table+" 执行的SQL为:"+querySql+" 具体错误信息为："+e); + } + if (dbUtilErrorCode == DBUtilErrorCode.MYSQL_QUERY_SELECT_PRI_ERROR && userName != null){ + return DataXException.asDataXException(dbUtilErrorCode,"用户名为："+userName+" 具体错误信息为："+e); + } + + return DataXException.asDataXException(dbUtilErrorCode,"执行的SQL为: "+querySql+" 具体错误信息为："+e); + } + + if (dataBaseType.equals(DataBaseType.Oracle)){ + DBUtilErrorCode dbUtilErrorCode = oracleQueryErrorAna(e.getMessage()); + if (dbUtilErrorCode == DBUtilErrorCode.ORACLE_QUERY_TABLE_NAME_ERROR && table != null){ + return DataXException.asDataXException(dbUtilErrorCode,"表名为："+table+" 执行的SQL为:"+querySql+" 具体错误信息为："+e); + } + if (dbUtilErrorCode == DBUtilErrorCode.ORACLE_QUERY_SELECT_PRI_ERROR){ + return DataXException.asDataXException(dbUtilErrorCode,"用户名为："+userName+" 具体错误信息为："+e); + } + + return DataXException.asDataXException(dbUtilErrorCode,"执行的SQL为: "+querySql+" 具体错误信息为："+e); + + } + + return DataXException.asDataXException(DBUtilErrorCode.SQL_EXECUTE_FAIL, "执行的SQL为: "+querySql+" 具体错误信息为："+e); + } + + public static DBUtilErrorCode mySqlQueryErrorAna(String e){ + if (e.contains(Constant.MYSQL_TABLE_NAME_ERR1) && e.contains(Constant.MYSQL_TABLE_NAME_ERR2)){ + return DBUtilErrorCode.MYSQL_QUERY_TABLE_NAME_ERROR; + }else if (e.contains(Constant.MYSQL_SELECT_PRI)){ + return DBUtilErrorCode.MYSQL_QUERY_SELECT_PRI_ERROR; + }else if (e.contains(Constant.MYSQL_COLUMN1) && e.contains(Constant.MYSQL_COLUMN2)){ + return DBUtilErrorCode.MYSQL_QUERY_COLUMN_ERROR; + }else if (e.contains(Constant.MYSQL_WHERE)){ + return DBUtilErrorCode.MYSQL_QUERY_SQL_ERROR; + } + return DBUtilErrorCode.READ_RECORD_FAIL; + } + + public static DBUtilErrorCode oracleQueryErrorAna(String e){ + if (e.contains(Constant.ORACLE_TABLE_NAME)){ + return DBUtilErrorCode.ORACLE_QUERY_TABLE_NAME_ERROR; + }else if (e.contains(Constant.ORACLE_SQL)){ + return DBUtilErrorCode.ORACLE_QUERY_SQL_ERROR; + }else if (e.contains(Constant.ORACLE_SELECT_PRI)){ + return DBUtilErrorCode.ORACLE_QUERY_SELECT_PRI_ERROR; + } + return DBUtilErrorCode.READ_RECORD_FAIL; + } + + public static DataXException asSqlParserException(DataBaseType dataBaseType, Exception e,String querySql){ + if (dataBaseType.equals(DataBaseType.MySql)){ + throw DataXException.asDataXException(DBUtilErrorCode.MYSQL_QUERY_SQL_PARSER_ERROR, "执行的SQL为:"+querySql+" 具体错误信息为：" + e); + } + if (dataBaseType.equals(DataBaseType.Oracle)){ + throw DataXException.asDataXException(DBUtilErrorCode.ORACLE_QUERY_SQL_PARSER_ERROR,"执行的SQL为:"+querySql+" 具体错误信息为：" +e); + } + throw DataXException.asDataXException(DBUtilErrorCode.READ_RECORD_FAIL,"执行的SQL为:"+querySql+" 具体错误信息为："+e); + } + + public static DataXException asPreSQLParserException(DataBaseType dataBaseType, Exception e,String querySql){ + if (dataBaseType.equals(DataBaseType.MySql)){ + throw DataXException.asDataXException(DBUtilErrorCode.MYSQL_PRE_SQL_ERROR, "执行的SQL为:"+querySql+" 具体错误信息为：" + e); + } + + if (dataBaseType.equals(DataBaseType.Oracle)){ + throw DataXException.asDataXException(DBUtilErrorCode.ORACLE_PRE_SQL_ERROR,"执行的SQL为:"+querySql+" 具体错误信息为：" +e); + } + throw DataXException.asDataXException(DBUtilErrorCode.READ_RECORD_FAIL,"执行的SQL为:"+querySql+" 具体错误信息为："+e); + } + + public static DataXException asPostSQLParserException(DataBaseType dataBaseType, Exception e,String querySql){ + if (dataBaseType.equals(DataBaseType.MySql)){ + throw DataXException.asDataXException(DBUtilErrorCode.MYSQL_POST_SQL_ERROR, "执行的SQL为:"+querySql+" 具体错误信息为：" + e); + } + + if (dataBaseType.equals(DataBaseType.Oracle)){ + throw DataXException.asDataXException(DBUtilErrorCode.ORACLE_POST_SQL_ERROR,"执行的SQL为:"+querySql+" 具体错误信息为：" +e); + } + throw DataXException.asDataXException(DBUtilErrorCode.READ_RECORD_FAIL,"执行的SQL为:"+querySql+" 具体错误信息为："+e); + } + + public static DataXException asInsertPriException(DataBaseType dataBaseType, String userName,String jdbcUrl){ + if (dataBaseType.equals(DataBaseType.MySql)){ + throw DataXException.asDataXException(DBUtilErrorCode.MYSQL_INSERT_ERROR, "用户名为:"+userName+" jdbcURL为："+jdbcUrl); + } + + if (dataBaseType.equals(DataBaseType.Oracle)){ + throw DataXException.asDataXException(DBUtilErrorCode.ORACLE_INSERT_ERROR,"用户名为:"+userName+" jdbcURL为："+jdbcUrl); + } + throw DataXException.asDataXException(DBUtilErrorCode.NO_INSERT_PRIVILEGE,"用户名为:"+userName+" jdbcURL为："+jdbcUrl); + } + + public static DataXException asDeletePriException(DataBaseType dataBaseType, String userName,String jdbcUrl){ + if (dataBaseType.equals(DataBaseType.MySql)){ + throw DataXException.asDataXException(DBUtilErrorCode.MYSQL_DELETE_ERROR, "用户名为:"+userName+" jdbcURL为："+jdbcUrl); + } + + if (dataBaseType.equals(DataBaseType.Oracle)){ + throw DataXException.asDataXException(DBUtilErrorCode.ORACLE_DELETE_ERROR,"用户名为:"+userName+" jdbcURL为："+jdbcUrl); + } + throw DataXException.asDataXException(DBUtilErrorCode.NO_DELETE_PRIVILEGE,"用户名为:"+userName+" jdbcURL为："+jdbcUrl); + } + + public static DataXException asSplitPKException(DataBaseType dataBaseType, Exception e,String splitSql,String splitPkID){ + if (dataBaseType.equals(DataBaseType.MySql)){ + + return DataXException.asDataXException(DBUtilErrorCode.MYSQL_SPLIT_PK_ERROR,"配置的SplitPK为: "+splitPkID+", 执行的SQL为: "+splitSql+" 具体错误信息为："+e); + } + + if (dataBaseType.equals(DataBaseType.Oracle)){ + return DataXException.asDataXException(DBUtilErrorCode.ORACLE_SPLIT_PK_ERROR,"配置的SplitPK为: "+splitPkID+", 执行的SQL为: "+splitSql+" 具体错误信息为："+e); + } + + return DataXException.asDataXException(DBUtilErrorCode.READ_RECORD_FAIL,splitSql+e); + } +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/RdbmsRangeSplitWrap.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/RdbmsRangeSplitWrap.java new file mode 100644 index 00000000..4390052b --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/RdbmsRangeSplitWrap.java @@ -0,0 +1,101 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.util; + +import com.alibaba.datax.common.util.RangeSplitUtil; +import org.apache.commons.lang3.StringUtils; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +public final class RdbmsRangeSplitWrap { + + public static List splitAndWrap(String left, String right, int expectSliceNumber, + String columnName, String quote, DataBaseType dataBaseType) { + String[] tempResult = RangeSplitUtil.doAsciiStringSplit(left, right, expectSliceNumber); + return RdbmsRangeSplitWrap.wrapRange(tempResult, columnName, quote, dataBaseType); + } + + // warn: do not use this method long->BigInteger + public static List splitAndWrap(long left, long right, int expectSliceNumber, String columnName) { + long[] tempResult = RangeSplitUtil.doLongSplit(left, right, expectSliceNumber); + return RdbmsRangeSplitWrap.wrapRange(tempResult, columnName); + } + + public static List splitAndWrap(BigInteger left, BigInteger right, int expectSliceNumber, String columnName) { + BigInteger[] tempResult = RangeSplitUtil.doBigIntegerSplit(left, right, expectSliceNumber); + return RdbmsRangeSplitWrap.wrapRange(tempResult, columnName); + } + + public static List wrapRange(long[] rangeResult, String columnName) { + String[] rangeStr = new String[rangeResult.length]; + for (int i = 0, len = rangeResult.length; i < len; i++) { + rangeStr[i] = String.valueOf(rangeResult[i]); + } + return wrapRange(rangeStr, columnName, "", null); + } + + public static List wrapRange(BigInteger[] rangeResult, String columnName) { + String[] rangeStr = new String[rangeResult.length]; + for (int i = 0, len = rangeResult.length; i < len; i++) { + rangeStr[i] = rangeResult[i].toString(); + } + return wrapRange(rangeStr, columnName, "", null); + } + + public static List wrapRange(String[] rangeResult, String columnName, + String quote, DataBaseType dataBaseType) { + if (null == rangeResult || rangeResult.length < 2) { + throw new IllegalArgumentException(String.format( + "Parameter rangeResult can not be null and its length can not <2. detail:rangeResult=[%s].", + StringUtils.join(rangeResult, ","))); + } + + List result = new ArrayList(); + + //TODO change to stringbuilder.append(..) + if (2 == rangeResult.length) { + result.add(String.format(" (%s%s%s <= %s AND %s <= %s%s%s) ", quote, quoteConstantValue(rangeResult[0], dataBaseType), + quote, columnName, columnName, quote, quoteConstantValue(rangeResult[1], dataBaseType), quote)); + return result; + } else { + for (int i = 0, len = rangeResult.length - 2; i < len; i++) { + result.add(String.format(" (%s%s%s <= %s AND %s < %s%s%s) ", quote, quoteConstantValue(rangeResult[i], dataBaseType), + quote, columnName, columnName, quote, quoteConstantValue(rangeResult[i + 1], dataBaseType), quote)); + } + + result.add(String.format(" (%s%s%s <= %s AND %s <= %s%s%s) ", quote, quoteConstantValue(rangeResult[rangeResult.length - 2], dataBaseType), + quote, columnName, columnName, quote, quoteConstantValue(rangeResult[rangeResult.length - 1], dataBaseType), quote)); + return result; + } + } + + public static String wrapFirstLastPoint(String firstPoint, String lastPoint, String columnName, + String quote, DataBaseType dataBaseType) { + return String.format(" ((%s < %s%s%s) OR (%s%s%s < %s)) ", columnName, quote, quoteConstantValue(firstPoint, dataBaseType), + quote, quote, quoteConstantValue(lastPoint, dataBaseType), quote, columnName); + } + + public static String wrapFirstLastPoint(Long firstPoint, Long lastPoint, String columnName) { + return wrapFirstLastPoint(firstPoint.toString(), lastPoint.toString(), columnName, "", null); + } + + public static String wrapFirstLastPoint(BigInteger firstPoint, BigInteger lastPoint, String columnName) { + return wrapFirstLastPoint(firstPoint.toString(), lastPoint.toString(), columnName, "", null); + } + + + private static String quoteConstantValue(String aString, DataBaseType dataBaseType) { + if (null == dataBaseType) { + return aString; + } + + if (dataBaseType.equals(DataBaseType.MySql)) { + return aString.replace("'", "''").replace("\\", "\\\\"); + } else if (dataBaseType.equals(DataBaseType.Oracle) || dataBaseType.equals(DataBaseType.SQLServer)) { + return aString.replace("'", "''"); + } else { + //TODO other type supported + return aString; + } + } +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/TableExpandUtil.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/TableExpandUtil.java new file mode 100644 index 00000000..600eadab --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/util/TableExpandUtil.java @@ -0,0 +1,83 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class TableExpandUtil { + + // schema.table[0-2]more + // 1 2 3 4 5 + public static Pattern pattern = Pattern + .compile("(\\w+\\.)?(\\w+)\\[(\\d+)-(\\d+)\\](.*)"); + + private TableExpandUtil() { + } + + /** + * Split the table string(Usually contains names of some tables) to a List + * that is formated. example: table[0-32] will be splitted into `table0`, + * `table1`, `table2`, ... ,`table32` in {@link List} + * + * @param tables + * a string contains table name(one or many). + * @return a split result of table name. + *

+ * TODO 删除参数 DataBaseType + */ + public static List splitTables(DataBaseType dataBaseType, + String tables) { + List splittedTables = new ArrayList(); + + String[] tableArrays = tables.split(","); + + String tableName = null; + for (String tableArray : tableArrays) { + Matcher matcher = pattern.matcher(tableArray.trim()); + if (!matcher.matches()) { + tableName = tableArray.trim(); + splittedTables.add(tableName); + } else { + String start = matcher.group(3).trim(); + String end = matcher.group(4).trim(); + String tmp = ""; + if (Integer.valueOf(start) > Integer.valueOf(end)) { + tmp = start; + start = end; + end = tmp; + } + int len = start.length(); + String schema = null; + for (int k = Integer.valueOf(start); k <= Integer.valueOf(end); k++) { + schema = (null == matcher.group(1)) ? "" : matcher.group(1) + .trim(); + if (start.startsWith("0")) { + tableName = schema + matcher.group(2).trim() + + String.format("%0" + len + "d", k) + + matcher.group(5).trim(); + splittedTables.add(tableName); + } else { + tableName = schema + matcher.group(2).trim() + + String.format("%d", k) + + matcher.group(5).trim(); + splittedTables.add(tableName); + } + } + } + } + return splittedTables; + } + + public static List expandTableConf(DataBaseType dataBaseType, + List tables) { + List parsedTables = new ArrayList(); + for (String table : tables) { + List splittedTables = splitTables(dataBaseType, table); + parsedTables.addAll(splittedTables); + } + + return parsedTables; + } + +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/CommonRdbmsWriter.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/CommonRdbmsWriter.java new file mode 100644 index 00000000..eecc25c7 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/CommonRdbmsWriter.java @@ -0,0 +1,566 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.alibaba.datax.common.util.Configuration; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtilErrorCode; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DataBaseType; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.RdbmsException; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.util.OriginalConfPretreatmentUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.util.WriterUtil; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Triple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; + +public class CommonRdbmsWriter { + + public static class Job { + private DataBaseType dataBaseType; + + private static final Logger LOG = LoggerFactory + .getLogger(Job.class); + + public Job(DataBaseType dataBaseType) { + this.dataBaseType = dataBaseType; + OriginalConfPretreatmentUtil.DATABASE_TYPE = this.dataBaseType; + } + + public void init(Configuration originalConfig) { + OriginalConfPretreatmentUtil.doPretreatment(originalConfig, this.dataBaseType); + + LOG.debug("After job init(), originalConfig now is:[\n{}\n]", + originalConfig.toJSON()); + } + + /*目前只支持MySQL Writer跟Oracle Writer;检查PreSQL跟PostSQL语法以及insert，delete权限*/ + public void writerPreCheck(Configuration originalConfig, DataBaseType dataBaseType) { + /*检查PreSql跟PostSql语句*/ + prePostSqlValid(originalConfig, dataBaseType); + /*检查insert 跟delete权限*/ + privilegeValid(originalConfig, dataBaseType); + } + + public void prePostSqlValid(Configuration originalConfig, DataBaseType dataBaseType) { + /*检查PreSql跟PostSql语句*/ + WriterUtil.preCheckPrePareSQL(originalConfig, dataBaseType); + WriterUtil.preCheckPostSQL(originalConfig, dataBaseType); + } + + public void privilegeValid(Configuration originalConfig, DataBaseType dataBaseType) { + /*检查insert 跟delete权限*/ + String username = originalConfig.getString(Key.USERNAME); + String password = originalConfig.getString(Key.PASSWORD); + List connections = originalConfig.getList(Constant.CONN_MARK, + Object.class); + + for (int i = 0, len = connections.size(); i < len; i++) { + Configuration connConf = Configuration.from(connections.get(i).toString()); + String jdbcUrl = connConf.getString(Key.JDBC_URL); + List expandedTables = connConf.getList(Key.TABLE, String.class); + boolean hasInsertPri = DBUtil.checkInsertPrivilege(dataBaseType, jdbcUrl, username, password, expandedTables); + + if (!hasInsertPri) { + throw RdbmsException.asInsertPriException(dataBaseType, originalConfig.getString(Key.USERNAME), jdbcUrl); + } + + if (DBUtil.needCheckDeletePrivilege(originalConfig)) { + boolean hasDeletePri = DBUtil.checkDeletePrivilege(dataBaseType, jdbcUrl, username, password, expandedTables); + if (!hasDeletePri) { + throw RdbmsException.asDeletePriException(dataBaseType, originalConfig.getString(Key.USERNAME), jdbcUrl); + } + } + } + } + + // 一般来说，是需要推迟到 task 中进行pre 的执行（单表情况例外） + public void prepare(Configuration originalConfig) { + int tableNumber = originalConfig.getInt(Constant.TABLE_NUMBER_MARK); + if (tableNumber == 1) { + String username = originalConfig.getString(Key.USERNAME); + String password = originalConfig.getString(Key.PASSWORD); + + List conns = originalConfig.getList(Constant.CONN_MARK, + Object.class); + Configuration connConf = Configuration.from(conns.get(0) + .toString()); + + // 这里的 jdbcUrl 已经 append 了合适后缀参数 + String jdbcUrl = connConf.getString(Key.JDBC_URL); + originalConfig.set(Key.JDBC_URL, jdbcUrl); + + String table = connConf.getList(Key.TABLE, String.class).get(0); + originalConfig.set(Key.TABLE, table); + + List preSqls = originalConfig.getList(Key.PRE_SQL, + String.class); + List renderedPreSqls = WriterUtil.renderPreOrPostSqls( + preSqls, table); + + originalConfig.remove(Constant.CONN_MARK); + if (null != renderedPreSqls && !renderedPreSqls.isEmpty()) { + // 说明有 preSql 配置，则此处删除掉 + originalConfig.remove(Key.PRE_SQL); + + Connection conn = DBUtil.getConnection(dataBaseType, + jdbcUrl, username, password); + LOG.info("Begin to execute preSqls:[{}]. context info:{}.", + StringUtils.join(renderedPreSqls, ";"), jdbcUrl); + + WriterUtil.executeSqls(conn, renderedPreSqls, jdbcUrl, dataBaseType); + DBUtil.closeDBResources(null, null, conn); + } + } + + LOG.debug("After job prepare(), originalConfig now is:[\n{}\n]", + originalConfig.toJSON()); + } + + public List split(Configuration originalConfig, + int mandatoryNumber) { + return WriterUtil.doSplit(originalConfig, mandatoryNumber); + } + + // 一般来说，是需要推迟到 task 中进行post 的执行（单表情况例外） + public void post(Configuration originalConfig) { + int tableNumber = originalConfig.getInt(Constant.TABLE_NUMBER_MARK); + if (tableNumber == 1) { + String username = originalConfig.getString(Key.USERNAME); + String password = originalConfig.getString(Key.PASSWORD); + + // 已经由 prepare 进行了appendJDBCSuffix处理 + String jdbcUrl = originalConfig.getString(Key.JDBC_URL); + + String table = originalConfig.getString(Key.TABLE); + + List postSqls = originalConfig.getList(Key.POST_SQL, + String.class); + List renderedPostSqls = WriterUtil.renderPreOrPostSqls( + postSqls, table); + + if (null != renderedPostSqls && !renderedPostSqls.isEmpty()) { + // 说明有 postSql 配置，则此处删除掉 + originalConfig.remove(Key.POST_SQL); + + Connection conn = DBUtil.getConnection(this.dataBaseType, + jdbcUrl, username, password); + + LOG.info( + "Begin to execute postSqls:[{}]. context info:{}.", + StringUtils.join(renderedPostSqls, ";"), jdbcUrl); + WriterUtil.executeSqls(conn, renderedPostSqls, jdbcUrl, dataBaseType); + DBUtil.closeDBResources(null, null, conn); + } + } + } + + public void destroy(Configuration originalConfig) { + } + + } + + public static class Task { + protected static final Logger LOG = LoggerFactory + .getLogger(Task.class); + + protected DataBaseType dataBaseType; + private static final String VALUE_HOLDER = "?"; + + protected String username; + protected String password; + protected String jdbcUrl; + protected String table; + protected List columns; + protected List preSqls; + protected List postSqls; + protected int batchSize; + protected int batchByteSize; + protected int columnNumber = 0; + protected TaskPluginCollector taskPluginCollector; + + // 作为日志显示信息时，需要附带的通用信息。比如信息所对应的数据库连接等信息，针对哪个表做的操作 + protected static String BASIC_MESSAGE; + + protected static String INSERT_OR_REPLACE_TEMPLATE; + + protected String writeRecordSql; + protected String writeMode; + protected boolean emptyAsNull; + protected Triple, List, List> resultSetMetaData; + + public Task(DataBaseType dataBaseType) { + this.dataBaseType = dataBaseType; + } + + public void init(Configuration writerSliceConfig) { + this.username = writerSliceConfig.getString(Key.USERNAME); + this.password = writerSliceConfig.getString(Key.PASSWORD); + this.jdbcUrl = writerSliceConfig.getString(Key.JDBC_URL); + + //ob10的处理 + if (this.jdbcUrl.startsWith(Constant.OB10_SPLIT_STRING) && this.dataBaseType == DataBaseType.MySql) { + String[] ss = this.jdbcUrl.split(Constant.OB10_SPLIT_STRING_PATTERN); + if (ss.length != 3) { + throw DataXException + .asDataXException( + DBUtilErrorCode.JDBC_OB10_ADDRESS_ERROR, "JDBC OB10格式错误，请联系askdatax"); + } + LOG.info("this is ob1_0 jdbc url."); + this.username = ss[1].trim() + ":" + this.username; + this.jdbcUrl = ss[2]; + LOG.info("this is ob1_0 jdbc url. user=" + this.username + " :url=" + this.jdbcUrl); + } + + this.table = writerSliceConfig.getString(Key.TABLE); + + this.columns = writerSliceConfig.getList(Key.COLUMN, String.class); + this.columnNumber = this.columns.size(); + + this.preSqls = writerSliceConfig.getList(Key.PRE_SQL, String.class); + this.postSqls = writerSliceConfig.getList(Key.POST_SQL, String.class); + this.batchSize = writerSliceConfig.getInt(Key.BATCH_SIZE, Constant.DEFAULT_BATCH_SIZE); + this.batchByteSize = writerSliceConfig.getInt(Key.BATCH_BYTE_SIZE, Constant.DEFAULT_BATCH_BYTE_SIZE); + + writeMode = writerSliceConfig.getString(Key.WRITE_MODE, "INSERT"); + emptyAsNull = writerSliceConfig.getBool(Key.EMPTY_AS_NULL, true); + INSERT_OR_REPLACE_TEMPLATE = writerSliceConfig.getString(Constant.INSERT_OR_REPLACE_TEMPLATE_MARK); + this.writeRecordSql = String.format(INSERT_OR_REPLACE_TEMPLATE, this.table); + + BASIC_MESSAGE = String.format("jdbcUrl:[%s], table:[%s]", + this.jdbcUrl, this.table); + } + + public void prepare(Configuration writerSliceConfig) { + Connection connection = DBUtil.getConnection(this.dataBaseType, + this.jdbcUrl, username, password); + + DBUtil.dealWithSessionConfig(connection, writerSliceConfig, + this.dataBaseType, BASIC_MESSAGE); + + int tableNumber = writerSliceConfig.getInt( + Constant.TABLE_NUMBER_MARK); + if (tableNumber != 1) { + LOG.info("Begin to execute preSqls:[{}]. context info:{}.", + StringUtils.join(this.preSqls, ";"), BASIC_MESSAGE); + WriterUtil.executeSqls(connection, this.preSqls, BASIC_MESSAGE, dataBaseType); + } + + DBUtil.closeDBResources(null, null, connection); + } + + public void startWriteWithConnection(RecordReceiver recordReceiver, + TaskPluginCollector taskPluginCollector, Connection connection) { + this.taskPluginCollector = taskPluginCollector; + + // 用于写入数据的时候的类型根据目的表字段类型转换 + this.resultSetMetaData = DBUtil.getColumnMetaData(connection, + this.table, StringUtils.join(this.columns, ",")); + // 写数据库的SQL语句 + calcWriteRecordSql(); + + List writeBuffer = new ArrayList(this.batchSize); + int bufferBytes = 0; + try { + Record record; + while ((record = recordReceiver.getFromReader()) != null) { + if (record.getColumnNumber() != this.columnNumber) { + // 源头读取字段列数与目的表字段写入列数不相等，直接报错 + throw DataXException + .asDataXException( + DBUtilErrorCode.CONF_ERROR, + String.format( + "列配置信息有错误. 因为您配置的任务中，源头读取字段数:%s 与 目的表要写入的字段数:%s 不相等. 请检查您的配置并作出修改.", + record.getColumnNumber(), + this.columnNumber)); + } + + writeBuffer.add(record); + bufferBytes += record.getMemorySize(); + + if (writeBuffer.size() >= batchSize || bufferBytes >= batchByteSize) { + doBatchInsert(connection, writeBuffer); + writeBuffer.clear(); + bufferBytes = 0; + } + } + if (!writeBuffer.isEmpty()) { + doBatchInsert(connection, writeBuffer); + writeBuffer.clear(); + bufferBytes = 0; + } + } catch (Exception e) { + throw DataXException.asDataXException( + DBUtilErrorCode.WRITE_DATA_ERROR, e); + } finally { + writeBuffer.clear(); + bufferBytes = 0; + DBUtil.closeDBResources(null, null, connection); + } + } + + // TODO 改用连接池，确保每次获取的连接都是可用的（注意：连接可能需要每次都初始化其 session） + public void startWrite(RecordReceiver recordReceiver, + Configuration writerSliceConfig, + TaskPluginCollector taskPluginCollector) { + Connection connection = DBUtil.getConnection(this.dataBaseType, + this.jdbcUrl, username, password); + DBUtil.dealWithSessionConfig(connection, writerSliceConfig, + this.dataBaseType, BASIC_MESSAGE); + startWriteWithConnection(recordReceiver, taskPluginCollector, connection); + } + + + public void post(Configuration writerSliceConfig) { + int tableNumber = writerSliceConfig.getInt(Constant.TABLE_NUMBER_MARK); + + boolean hasPostSql = (this.postSqls != null && this.postSqls.size() > 0); + if (tableNumber == 1 || !hasPostSql) { + return; + } + + Connection connection = DBUtil.getConnection(this.dataBaseType, + this.jdbcUrl, username, password); + + LOG.info("Begin to execute postSqls:[{}]. context info:{}.", + StringUtils.join(this.postSqls, ";"), BASIC_MESSAGE); + WriterUtil.executeSqls(connection, this.postSqls, BASIC_MESSAGE, dataBaseType); + DBUtil.closeDBResources(null, null, connection); + } + + public void destroy(Configuration writerSliceConfig) { + } + + protected void doBatchInsert(Connection connection, List buffer) + throws SQLException { + PreparedStatement preparedStatement = null; + try { + connection.setAutoCommit(false); + preparedStatement = connection.prepareStatement(this.writeRecordSql); + for (Record record : buffer) { + preparedStatement = fillPreparedStatement(preparedStatement, record); + preparedStatement.addBatch(); + } + preparedStatement.executeBatch(); + connection.commit(); + } catch (SQLException e) { + LOG.warn("回滚此次写入, 采用每次写入一行方式提交. 因为:" + e.getMessage()); + connection.rollback(); + doOneInsert(connection, buffer); + } catch (Exception e) { + throw DataXException.asDataXException( + DBUtilErrorCode.WRITE_DATA_ERROR, e); + } finally { + DBUtil.closeDBResources(preparedStatement, null); + } + } + + protected void doOneInsert(Connection connection, List buffer) { + PreparedStatement preparedStatement = null; + try { + connection.setAutoCommit(true); + preparedStatement = connection + .prepareStatement(this.writeRecordSql); + + for (Record record : buffer) { + try { + preparedStatement = fillPreparedStatement( + preparedStatement, record); + preparedStatement.execute(); + } catch (SQLException e) { + LOG.debug(e.toString()); + + this.taskPluginCollector.collectDirtyRecord(record, e); + } finally { + // 最后不要忘了关闭 preparedStatement + preparedStatement.clearParameters(); + } + } + } catch (Exception e) { + throw DataXException.asDataXException( + DBUtilErrorCode.WRITE_DATA_ERROR, e); + } finally { + DBUtil.closeDBResources(preparedStatement, null); + } + } + + // 直接使用了两个类变量：columnNumber,resultSetMetaData + protected PreparedStatement fillPreparedStatement(PreparedStatement preparedStatement, Record record) + throws SQLException { + for (int i = 0; i < this.columnNumber; i++) { + int columnSqltype = this.resultSetMetaData.getMiddle().get(i); + preparedStatement = fillPreparedStatementColumnType(preparedStatement, i, columnSqltype, record.getColumn(i)); + } + + return preparedStatement; + } + + protected PreparedStatement fillPreparedStatementColumnType(PreparedStatement preparedStatement, int columnIndex, int columnSqltype, Column column) throws SQLException { + java.util.Date utilDate; + switch (columnSqltype) { + case Types.CHAR: + case Types.NCHAR: + case Types.CLOB: + case Types.NCLOB: + case Types.VARCHAR: + case Types.LONGVARCHAR: + case Types.NVARCHAR: + case Types.LONGNVARCHAR: + preparedStatement.setString(columnIndex + 1, column + .asString()); + break; + + case Types.SMALLINT: + case Types.INTEGER: + case Types.BIGINT: + case Types.NUMERIC: + case Types.DECIMAL: + case Types.FLOAT: + case Types.REAL: + case Types.DOUBLE: + String strValue = column.asString(); + if (emptyAsNull && "".equals(strValue)) { + preparedStatement.setString(columnIndex + 1, null); + } else { + preparedStatement.setString(columnIndex + 1, strValue); + } + break; + + //tinyint is a little special in some database like mysql {boolean->tinyint(1)} + case Types.TINYINT: + Long longValue = column.asLong(); + if (null == longValue) { + preparedStatement.setString(columnIndex + 1, null); + } else { + preparedStatement.setString(columnIndex + 1, longValue.toString()); + } + break; + + // for mysql bug, see http://bugs.mysql.com/bug.php?id=35115 + case Types.DATE: + if (this.resultSetMetaData.getRight().get(columnIndex) + .equalsIgnoreCase("year")) { + if (column.asBigInteger() == null) { + preparedStatement.setString(columnIndex + 1, null); + } else { + preparedStatement.setInt(columnIndex + 1, column.asBigInteger().intValue()); + } + } else { + java.sql.Date sqlDate = null; + try { + utilDate = column.asDate(); + } catch (DataXException e) { + throw new SQLException(String.format( + "Date 类型转换错误：[%s]", column)); + } + + if (null != utilDate) { + sqlDate = new java.sql.Date(utilDate.getTime()); + } + preparedStatement.setDate(columnIndex + 1, sqlDate); + } + break; + + case Types.TIME: + java.sql.Time sqlTime = null; + try { + utilDate = column.asDate(); + } catch (DataXException e) { + throw new SQLException(String.format( + "TIME 类型转换错误：[%s]", column)); + } + + if (null != utilDate) { + sqlTime = new java.sql.Time(utilDate.getTime()); + } + preparedStatement.setTime(columnIndex + 1, sqlTime); + break; + + case Types.TIMESTAMP: + java.sql.Timestamp sqlTimestamp = null; + try { + utilDate = column.asDate(); + } catch (DataXException e) { + throw new SQLException(String.format( + "TIMESTAMP 类型转换错误：[%s]", column)); + } + + if (null != utilDate) { + sqlTimestamp = new java.sql.Timestamp( + utilDate.getTime()); + } + preparedStatement.setTimestamp(columnIndex + 1, sqlTimestamp); + break; + + case Types.BINARY: + case Types.VARBINARY: + case Types.BLOB: + case Types.LONGVARBINARY: + preparedStatement.setBytes(columnIndex + 1, column + .asBytes()); + break; + + case Types.BOOLEAN: + preparedStatement.setString(columnIndex + 1, column.asString()); + break; + + // warn: bit(1) -> Types.BIT 可使用setBoolean + // warn: bit(>1) -> Types.VARBINARY 可使用setBytes + case Types.BIT: + if (this.dataBaseType == DataBaseType.MySql) { + preparedStatement.setBoolean(columnIndex + 1, column.asBoolean()); + } else { + preparedStatement.setString(columnIndex + 1, column.asString()); + } + break; + default: + throw DataXException + .asDataXException( + DBUtilErrorCode.UNSUPPORTED_TYPE, + String.format( + "您的配置文件中的列配置信息有误. 因为DataX 不支持数据库写入这种字段类型. 字段名:[%s], 字段类型:[%d], 字段Java类型:[%s]. 请修改表中该字段的类型或者不同步该字段.", + this.resultSetMetaData.getLeft() + .get(columnIndex), + this.resultSetMetaData.getMiddle() + .get(columnIndex), + this.resultSetMetaData.getRight() + .get(columnIndex))); + } + return preparedStatement; + } + + private void calcWriteRecordSql() { + if (!VALUE_HOLDER.equals(calcValueHolder(""))) { + List valueHolders = new ArrayList(columnNumber); + for (int i = 0; i < columns.size(); i++) { + String type = resultSetMetaData.getRight().get(i); + valueHolders.add(calcValueHolder(type)); + } + + boolean forceUseUpdate = false; + //ob10的处理 + if (dataBaseType != null && dataBaseType == DataBaseType.MySql && OriginalConfPretreatmentUtil.isOB10(jdbcUrl)) { + forceUseUpdate = true; + } + + INSERT_OR_REPLACE_TEMPLATE = WriterUtil.getWriteTemplate(columns, valueHolders, writeMode, dataBaseType, forceUseUpdate); + writeRecordSql = String.format(INSERT_OR_REPLACE_TEMPLATE, this.table); + } + } + + protected String calcValueHolder(String columnType) { + return VALUE_HOLDER; + } + } +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/Constant.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/Constant.java new file mode 100644 index 00000000..67d89d90 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/Constant.java @@ -0,0 +1,22 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer; + +/** + * 用于插件解析用户配置时，需要进行标识（MARK）的常量的声明. + */ +public final class Constant { + public static final int DEFAULT_BATCH_SIZE = 2048; + + public static final int DEFAULT_BATCH_BYTE_SIZE = 32 * 1024 * 1024; + + public static String TABLE_NAME_PLACEHOLDER = "@table"; + + public static String CONN_MARK = "connection"; + + public static String TABLE_NUMBER_MARK = "tableNumber"; + + public static String INSERT_OR_REPLACE_TEMPLATE_MARK = "insertOrReplaceTemplate"; + + public static final String OB10_SPLIT_STRING = "||_dsc_ob10_dsc_||"; + public static final String OB10_SPLIT_STRING_PATTERN = "\\|\\|_dsc_ob10_dsc_\\|\\|"; + +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/Key.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/Key.java new file mode 100644 index 00000000..f64adbf0 --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/Key.java @@ -0,0 +1,40 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer; + +public final class Key { + public final static String JDBC_URL = "jdbcUrl"; + + public final static String USERNAME = "username"; + + public final static String PASSWORD = "password"; + + public final static String TABLE = "table"; + + public final static String COLUMN = "column"; + + //可选值为：insert,replace，默认为 insert （mysql 支持，oracle 没用 replace 机制，只能 insert,oracle 可以不暴露这个参数） + public final static String WRITE_MODE = "writeMode"; + + public final static String PRE_SQL = "preSql"; + + public final static String POST_SQL = "postSql"; + + public final static String TDDL_APP_NAME = "appName"; + + //默认值：256 + public final static String BATCH_SIZE = "batchSize"; + + //默认值：32m + public final static String BATCH_BYTE_SIZE = "batchByteSize"; + + public final static String EMPTY_AS_NULL = "emptyAsNull"; + + public final static String DB_NAME_PATTERN = "dbNamePattern"; + + public final static String DB_RULE = "dbRule"; + + public final static String TABLE_NAME_PATTERN = "tableNamePattern"; + + public final static String TABLE_RULE = "tableRule"; + + public final static String DRYRUN = "dryRun"; +} \ No newline at end of file diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/MysqlWriterErrorCode.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/MysqlWriterErrorCode.java new file mode 100644 index 00000000..ad273a8a --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/MysqlWriterErrorCode.java @@ -0,0 +1,32 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer; + +import com.alibaba.datax.common.spi.ErrorCode; + +//TODO 后续考虑与 util 包种的 DBUTilErrorCode 做合并.（区分读和写的错误码） +public enum MysqlWriterErrorCode implements ErrorCode { + ; + + private final String code; + private final String describe; + + private MysqlWriterErrorCode(String code, String describe) { + this.code = code; + this.describe = describe; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.describe; + } + + @Override + public String toString() { + return String.format("Code:[%s], Describe:[%s]. ", this.code, + this.describe); + } +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/util/OriginalConfPretreatmentUtil.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/util/OriginalConfPretreatmentUtil.java new file mode 100644 index 00000000..6c06932b --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/util/OriginalConfPretreatmentUtil.java @@ -0,0 +1,183 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.util; + +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.common.util.ListUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.*; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.Constant; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.Key; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +// 原生配置预处理 +public final class OriginalConfPretreatmentUtil { + private static final Logger LOG = LoggerFactory + .getLogger(OriginalConfPretreatmentUtil.class); + + public static DataBaseType DATABASE_TYPE; + +// public static void doPretreatment(Configuration originalConfig) { +// doPretreatment(originalConfig,null); +// } + + public static void doPretreatment(Configuration originalConfig, DataBaseType dataBaseType) { + // 检查 username/password 配置（必填） + originalConfig.getNecessaryValue(Key.USERNAME, DBUtilErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(Key.PASSWORD, DBUtilErrorCode.REQUIRED_VALUE); + // + doCheckBatchSize(originalConfig); + // + simplifyConf(originalConfig); + // + dealColumnConf(originalConfig); + dealWriteMode(originalConfig, dataBaseType); + } + + public static void doCheckBatchSize(Configuration originalConfig) { + // 检查batchSize 配置（选填，如果未填写，则设置为默认值） + int batchSize = originalConfig.getInt(Key.BATCH_SIZE, Constant.DEFAULT_BATCH_SIZE); + if (batchSize < 1) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE, String.format( + "您的batchSize配置有误. 您所配置的写入数据库表的 batchSize:%s 不能小于1. 推荐配置范围为：[100-1000], 该值越大, 内存溢出可能性越大. 请检查您的配置并作出修改.", + batchSize)); + } + + originalConfig.set(Key.BATCH_SIZE, batchSize); + } + + public static void simplifyConf(Configuration originalConfig) { + // + List connections = originalConfig.getList(Constant.CONN_MARK, Object.class); + // + int tableNum = 0; + for (int i = 0, len = connections.size(); i < len; i++) { + Configuration connConf = Configuration.from(connections.get(i).toString()); + + String jdbcUrl = connConf.getString(Key.JDBC_URL); + if (StringUtils.isBlank(jdbcUrl)) { + throw DataXException.asDataXException(DBUtilErrorCode.REQUIRED_VALUE, "您未配置的写入数据库表的 jdbcUrl."); + } + // 增加jdbc url后缀，其实只有mysql + jdbcUrl = DATABASE_TYPE.appendJDBCSuffixForReader(jdbcUrl); + originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, i, Key.JDBC_URL), + jdbcUrl); + // table列表 + List tables = connConf.getList(Key.TABLE, String.class); + if (null == tables || tables.isEmpty()) { + throw DataXException.asDataXException(DBUtilErrorCode.REQUIRED_VALUE, + "您未配置写入数据库表的表名称. 根据配置DataX找不到您配置的表. 请检查您的配置并作出修改."); + } + // 对每一个connection 上配置的table 项进行解析 + List expandedTables = TableExpandUtil.expandTableConf(DATABASE_TYPE, tables); + + if (null == expandedTables || expandedTables.isEmpty()) { + throw DataXException.asDataXException(DBUtilErrorCode.CONF_ERROR, + "您配置的写入数据库表名称错误. DataX找不到您配置的表，请检查您的配置并作出修改."); + } + + tableNum += expandedTables.size(); + + originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, + i, Key.TABLE), expandedTables); + } + + originalConfig.set(Constant.TABLE_NUMBER_MARK, tableNum); + } + + public static void dealColumnConf(Configuration originalConfig, ConnectionFactory connectionFactory, String oneTable) { + List userConfiguredColumns = originalConfig.getList(Key.COLUMN, String.class); + if (null == userConfiguredColumns || userConfiguredColumns.isEmpty()) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE, + "您的配置文件中的列配置信息有误. 因为您未配置写入数据库表的列名称，DataX获取不到列信息. 请检查您的配置并作出修改."); + } else { + boolean isPreCheck = originalConfig.getBool(Key.DRYRUN, false); + List allColumns; + if (isPreCheck){ + allColumns = DBUtil.getTableColumnsByConn(DATABASE_TYPE,connectionFactory.getConnecttionWithoutRetry(), oneTable, connectionFactory.getConnectionInfo()); + }else{ + allColumns = DBUtil.getTableColumnsByConn(DATABASE_TYPE,connectionFactory.getConnecttion(), oneTable, connectionFactory.getConnectionInfo()); + } + + LOG.info("table:[{}] all columns:[\n{}\n].", oneTable, + StringUtils.join(allColumns, ",")); + + if (1 == userConfiguredColumns.size() && "*".equals(userConfiguredColumns.get(0))) { + LOG.warn("您的配置文件中的列配置信息存在风险. 因为您配置的写入数据库表的列为*，当您的表字段个数、类型有变动时，可能影响任务正确性甚至会运行出错。请检查您的配置并作出修改."); + + // 回填其值，需要以 String 的方式转交后续处理 + originalConfig.set(Key.COLUMN, allColumns); + } else if (userConfiguredColumns.size() > allColumns.size()) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE, + String.format("您的配置文件中的列配置信息有误. 因为您所配置的写入数据库表的字段个数:%s 大于目的表的总字段总个数:%s. 请检查您的配置并作出修改.", + userConfiguredColumns.size(), allColumns.size())); + } else { + // 确保用户配置的 column 不重复 + ListUtil.makeSureNoValueDuplicate(userConfiguredColumns, false); + + // 检查列是否都为数据库表中正确的列（通过执行一次 select column from table 进行判断） + DBUtil.getColumnMetaData(connectionFactory.getConnecttion(), oneTable,StringUtils.join(userConfiguredColumns, ",")); + } + } + } + + public static void dealColumnConf(Configuration originalConfig) { + String jdbcUrl = originalConfig.getString(String.format("%s[0].%s", + Constant.CONN_MARK, Key.JDBC_URL)); + + String username = originalConfig.getString(Key.USERNAME); + String password = originalConfig.getString(Key.PASSWORD); + String oneTable = originalConfig.getString(String.format( + "%s[0].%s[0]", Constant.CONN_MARK, Key.TABLE)); + + JdbcConnectionFactory jdbcConnectionFactory = new JdbcConnectionFactory(DATABASE_TYPE, jdbcUrl, username, password); + dealColumnConf(originalConfig, jdbcConnectionFactory, oneTable); + } + + public static void dealWriteMode(Configuration originalConfig, DataBaseType dataBaseType) { + List columns = originalConfig.getList(Key.COLUMN, String.class); + + String jdbcUrl = originalConfig.getString(String.format("%s[0].%s", + Constant.CONN_MARK, Key.JDBC_URL, String.class)); + + // 默认为：insert 方式 + String writeMode = originalConfig.getString(Key.WRITE_MODE, "INSERT"); + + List valueHolders = new ArrayList(columns.size()); + for (int i = 0; i < columns.size(); i++) { + valueHolders.add("?"); + } + + boolean forceUseUpdate = false; + //ob10的处理 + if (dataBaseType == DataBaseType.MySql && isOB10(jdbcUrl)) { + forceUseUpdate = true; + } + + String writeDataSqlTemplate = WriterUtil.getWriteTemplate(columns, valueHolders, writeMode,dataBaseType, forceUseUpdate); + + LOG.info("Write data [\n{}\n], which jdbcUrl like:[{}]", writeDataSqlTemplate, jdbcUrl); + + originalConfig.set(Constant.INSERT_OR_REPLACE_TEMPLATE_MARK, writeDataSqlTemplate); + } + + // 支持自家oceanbase + public static boolean isOB10(String jdbcUrl) { + //ob10的处理 + if (jdbcUrl.startsWith(com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.Constant.OB10_SPLIT_STRING)) { + String[] ss = jdbcUrl.split(com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.Constant.OB10_SPLIT_STRING_PATTERN); + if (ss.length != 3) { + throw DataXException + .asDataXException( + DBUtilErrorCode.JDBC_OB10_ADDRESS_ERROR, "JDBC OB10格式错误，请联系askdatax"); + } + return true; + } + return false; + } + +} diff --git a/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/util/WriterUtil.java b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/util/WriterUtil.java new file mode 100644 index 00000000..c2e334ce --- /dev/null +++ b/rdb2graph/rdb2grpah-rdbms-util/src/main/java/com/leehom/arch/datax/plugin/rdb2graph/rdbms/writer/util/WriterUtil.java @@ -0,0 +1,219 @@ +package com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.util; + +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.druid.sql.parser.ParserException; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtil; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DBUtilErrorCode; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.DataBaseType; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.util.RdbmsException; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.Constant; +import com.leehom.arch.datax.plugin.rdb2graph.rdbms.writer.Key; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.*; + +public final class WriterUtil { + private static final Logger LOG = LoggerFactory.getLogger(WriterUtil.class); + + //TODO 切分报错 + public static List doSplit(Configuration simplifiedConf, + int adviceNumber) { + + List splitResultConfigs = new ArrayList(); + + int tableNumber = simplifiedConf.getInt(Constant.TABLE_NUMBER_MARK); + + //处理单表的情况 + if (tableNumber == 1) { + //由于在之前的 master prepare 中已经把 table,jdbcUrl 提取出来，所以这里处理十分简单 + for (int j = 0; j < adviceNumber; j++) { + splitResultConfigs.add(simplifiedConf.clone()); + } + + return splitResultConfigs; + } + + if (tableNumber != adviceNumber) { + throw DataXException.asDataXException(DBUtilErrorCode.CONF_ERROR, + String.format("您的配置文件中的列配置信息有误. 您要写入的目的端的表个数是:%s , 但是根据系统建议需要切分的份数是：%s. 请检查您的配置并作出修改.", + tableNumber, adviceNumber)); + } + + String jdbcUrl; + List preSqls = simplifiedConf.getList(Key.PRE_SQL, String.class); + List postSqls = simplifiedConf.getList(Key.POST_SQL, String.class); + + List conns = simplifiedConf.getList(Constant.CONN_MARK, + Object.class); + + for (Object conn : conns) { + Configuration sliceConfig = simplifiedConf.clone(); + + Configuration connConf = Configuration.from(conn.toString()); + jdbcUrl = connConf.getString(Key.JDBC_URL); + sliceConfig.set(Key.JDBC_URL, jdbcUrl); + + sliceConfig.remove(Constant.CONN_MARK); + + List tables = connConf.getList(Key.TABLE, String.class); + + for (String table : tables) { + Configuration tempSlice = sliceConfig.clone(); + tempSlice.set(Key.TABLE, table); + tempSlice.set(Key.PRE_SQL, renderPreOrPostSqls(preSqls, table)); + tempSlice.set(Key.POST_SQL, renderPreOrPostSqls(postSqls, table)); + + splitResultConfigs.add(tempSlice); + } + + } + + return splitResultConfigs; + } + + public static List renderPreOrPostSqls(List preOrPostSqls, String tableName) { + if (null == preOrPostSqls) { + return Collections.emptyList(); + } + + List renderedSqls = new ArrayList(); + for (String sql : preOrPostSqls) { + //preSql为空时，不加入执行队列 + if (StringUtils.isNotBlank(sql)) { + renderedSqls.add(sql.replace(Constant.TABLE_NAME_PLACEHOLDER, tableName)); + } + } + + return renderedSqls; + } + + public static void executeSqls(Connection conn, List sqls, String basicMessage,DataBaseType dataBaseType) { + Statement stmt = null; + String currentSql = null; + try { + stmt = conn.createStatement(); + for (String sql : sqls) { + currentSql = sql; + DBUtil.executeSqlWithoutResultSet(stmt, sql); + } + } catch (Exception e) { + throw RdbmsException.asQueryException(dataBaseType,e,currentSql,null,null); + } finally { + DBUtil.closeDBResources(null, stmt, null); + } + } + + public static String getWriteTemplate(List columnHolders, List valueHolders, String writeMode, DataBaseType dataBaseType, boolean forceUseUpdate) { + boolean isWriteModeLegal = writeMode.trim().toLowerCase().startsWith("insert") + || writeMode.trim().toLowerCase().startsWith("replace") + || writeMode.trim().toLowerCase().startsWith("update"); + + if (!isWriteModeLegal) { + throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE, + String.format("您所配置的 writeMode:%s 错误. 因为DataX 目前仅支持replace,update 或 insert 方式. 请检查您的配置并作出修改.", writeMode)); + } + // && writeMode.trim().toLowerCase().startsWith("replace") + String writeDataSqlTemplate; + if (forceUseUpdate || + ((dataBaseType == DataBaseType.MySql || dataBaseType == DataBaseType.Tddl) && writeMode.trim().toLowerCase().startsWith("update")) + ) { + //update只在mysql下使用 + + writeDataSqlTemplate = new StringBuilder() + .append("INSERT INTO %s (").append(StringUtils.join(columnHolders, ",")) + .append(") VALUES(").append(StringUtils.join(valueHolders, ",")) + .append(")") + .append(onDuplicateKeyUpdateString(columnHolders)) + .toString(); + } else { + + //这里是保护,如果其他错误的使用了update,需要更换为replace + if (writeMode.trim().toLowerCase().startsWith("update")) { + writeMode = "replace"; + } + writeDataSqlTemplate = new StringBuilder().append(writeMode) + .append(" INTO %s (").append(StringUtils.join(columnHolders, ",")) + .append(") VALUES(").append(StringUtils.join(valueHolders, ",")) + .append(")").toString(); + } + + return writeDataSqlTemplate; + } + + public static String onDuplicateKeyUpdateString(List columnHolders){ + if (columnHolders == null || columnHolders.size() < 1) { + return ""; + } + StringBuilder sb = new StringBuilder(); + sb.append(" ON DUPLICATE KEY UPDATE "); + boolean first = true; + for(String column:columnHolders){ + if(!first){ + sb.append(","); + }else{ + first = false; + } + sb.append(column); + sb.append("=VALUES("); + sb.append(column); + sb.append(")"); + } + + return sb.toString(); + } + + public static void preCheckPrePareSQL(Configuration originalConfig, DataBaseType type) { + List conns = originalConfig.getList(Constant.CONN_MARK, Object.class); + Configuration connConf = Configuration.from(conns.get(0).toString()); + String table = connConf.getList(Key.TABLE, String.class).get(0); + + List preSqls = originalConfig.getList(Key.PRE_SQL, + String.class); + List renderedPreSqls = WriterUtil.renderPreOrPostSqls( + preSqls, table); + + if (null != renderedPreSqls && !renderedPreSqls.isEmpty()) { + LOG.info("Begin to preCheck preSqls:[{}].", + StringUtils.join(renderedPreSqls, ";")); + for(String sql : renderedPreSqls) { + try{ + DBUtil.sqlValid(sql, type); + }catch(ParserException e) { + throw RdbmsException.asPreSQLParserException(type,e,sql); + } + } + } + } + + public static void preCheckPostSQL(Configuration originalConfig, DataBaseType type) { + List conns = originalConfig.getList(Constant.CONN_MARK, Object.class); + Configuration connConf = Configuration.from(conns.get(0).toString()); + String table = connConf.getList(Key.TABLE, String.class).get(0); + + List postSqls = originalConfig.getList(Key.POST_SQL, + String.class); + List renderedPostSqls = WriterUtil.renderPreOrPostSqls( + postSqls, table); + if (null != renderedPostSqls && !renderedPostSqls.isEmpty()) { + + LOG.info("Begin to preCheck postSqls:[{}].", + StringUtils.join(renderedPostSqls, ";")); + for(String sql : renderedPostSqls) { + try{ + DBUtil.sqlValid(sql, type); + }catch(ParserException e){ + throw RdbmsException.asPostSQLParserException(type,e,sql); + } + + } + } + } + + +}