mirror of
https://github.com/alibaba/DataX.git
synced 2025-05-02 07:21:48 +08:00
[feature][plugin][sybase] datax support sybase plugins. #1780
add reader plugin
This commit is contained in:
xiaopengm
parent
65a880b852
commit
e158e5845b
@ -25,7 +25,9 @@ public enum DataBaseType {
|
|||||||
Oscar("oscar", "com.oscar.Driver"),
|
Oscar("oscar", "com.oscar.Driver"),
|
||||||
OceanBase("oceanbase", "com.alipay.oceanbase.jdbc.Driver"),
|
OceanBase("oceanbase", "com.alipay.oceanbase.jdbc.Driver"),
|
||||||
StarRocks("starrocks", "com.mysql.jdbc.Driver"),
|
StarRocks("starrocks", "com.mysql.jdbc.Driver"),
|
||||||
Databend("databend", "com.databend.jdbc.DatabendDriver");
|
Databend("databend", "com.databend.jdbc.DatabendDriver"),
|
||||||
|
Sybase("sybase", "com.sybase.jdbc4.jdbc.SybDriver");
|
||||||
|
|
||||||
|
|
||||||
private String typeName;
|
private String typeName;
|
||||||
private String driverClassName;
|
private String driverClassName;
|
||||||
|
|||||||
@ -233,12 +233,12 @@ SybaseReader插件实现了从Sybase读取数据。在底层实现上，SybaseRe
|
|||||||
|
|
||||||
| DataX 内部类型| Sybase 数据类型 |
|
| DataX 内部类型| Sybase 数据类型 |
|
||||||
| -------- | ----- |
|
| -------- | ----- |
|
||||||
| Long |NUMBER,INTEGER,INT,SMALLINT|
|
| Long |Tinyint,Smallint,Int,Money,Smallmoney|
|
||||||
| Double |NUMERIC,DECIMAL,FLOAT,DOUBLE PRECISION,REAL|
|
| Double |Float,Real,Numeric,Decimal|
|
||||||
| String |LONG,CHAR,NCHAR,VARCHAR,VARCHAR2,NVARCHAR2,CLOB,NCLOB,CHARACTER,CHARACTER VARYING,CHAR VARYING,NATIONAL CHARACTER,NATIONAL CHAR,NATIONAL CHARACTER VARYING,NATIONAL CHAR VARYING,NCHAR VARYING |
|
| String |Char,Varchar,Nchar,Nvarchar,Text|
|
||||||
| Date |TIMESTAMP,DATE |
|
| Date |Timestamp,Datetime,Smalldatetime|
|
||||||
| Boolean |bit, bool |
|
| Boolean |bit, bool|
|
||||||
| Bytes |BLOB,BFILE,RAW,LONG RAW |
|
| Bytes |Binary,Varbinary,Image|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
105
sybasereader/pom.xml
Normal file
105
sybasereader/pom.xml
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<parent>
|
||||||
|
<artifactId>datax-all</artifactId>
|
||||||
|
<groupId>com.alibaba.datax</groupId>
|
||||||
|
<version>0.0.1-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<artifactId>sybasereader</artifactId>
|
||||||
|
<name>sybasereader</name>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.alibaba.datax</groupId>
|
||||||
|
<artifactId>datax-common</artifactId>
|
||||||
|
<version>${datax-project-version}</version>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>slf4j-log4j12</artifactId>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>ch.qos.logback</groupId>
|
||||||
|
<artifactId>logback-classic</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.alibaba.datax</groupId>
|
||||||
|
<artifactId>plugin-rdbms-util</artifactId>
|
||||||
|
<version>${datax-project-version}</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.oracle</groupId>
|
||||||
|
<artifactId>ojdbc6</artifactId>
|
||||||
|
<version>11.2.0.3</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.alibaba.datax</groupId>
|
||||||
|
<artifactId>datax-common</artifactId>
|
||||||
|
<version>0.0.1-SNAPSHOT</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.sybase.jconnect</groupId>
|
||||||
|
<artifactId>jconn4</artifactId>
|
||||||
|
<version>16.0</version>
|
||||||
|
<scope>system</scope>
|
||||||
|
<systemPath>${project.basedir}/libs/jconn4-16.0.jar</systemPath>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<!-- compiler plugin -->
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<source>${jdk-version}</source>
|
||||||
|
<target>${jdk-version}</target>
|
||||||
|
<encoding>${project-sourceEncoding}</encoding>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<!-- assembly plugin -->
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<descriptors>
|
||||||
|
<descriptor>src/main/assembly/package.xml</descriptor>
|
||||||
|
</descriptors>
|
||||||
|
<finalName>datax</finalName>
|
||||||
|
</configuration>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>dwzip</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
|
||||||
|
</project>
|
||||||
35
sybasereader/src/main/assembly/package.xml
Executable file
35
sybasereader/src/main/assembly/package.xml
Executable file
@ -0,0 +1,35 @@
|
|||||||
|
<assembly
|
||||||
|
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||||
|
<id></id>
|
||||||
|
<formats>
|
||||||
|
<format>dir</format>
|
||||||
|
</formats>
|
||||||
|
<includeBaseDirectory>false</includeBaseDirectory>
|
||||||
|
<fileSets>
|
||||||
|
<fileSet>
|
||||||
|
<directory>src/main/resources</directory>
|
||||||
|
<includes>
|
||||||
|
<include>plugin.json</include>
|
||||||
|
<include>plugin_job_template.json</include>
|
||||||
|
</includes>
|
||||||
|
<outputDirectory>plugin/reader/oraclereader</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>target/</directory>
|
||||||
|
<includes>
|
||||||
|
<include>oraclereader-0.0.1-SNAPSHOT.jar</include>
|
||||||
|
</includes>
|
||||||
|
<outputDirectory>plugin/reader/oraclereader</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
|
</fileSets>
|
||||||
|
|
||||||
|
<dependencySets>
|
||||||
|
<dependencySet>
|
||||||
|
<useProjectArtifact>false</useProjectArtifact>
|
||||||
|
<outputDirectory>plugin/reader/oraclereader/libs</outputDirectory>
|
||||||
|
<scope>runtime</scope>
|
||||||
|
</dependencySet>
|
||||||
|
</dependencySets>
|
||||||
|
</assembly>
|
||||||
@ -0,0 +1,7 @@
|
|||||||
|
package com.alibaba.datax.plugin.reader.sybasereader;
|
||||||
|
|
||||||
|
public class Constants {
|
||||||
|
|
||||||
|
public static final int DEFAULT_FETCH_SIZE = 1024;
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,107 @@
|
|||||||
|
package com.alibaba.datax.plugin.reader.sybasereader;
|
||||||
|
|
||||||
|
import com.alibaba.datax.common.plugin.RecordSender;
|
||||||
|
import com.alibaba.datax.common.spi.Reader;
|
||||||
|
import com.alibaba.datax.common.util.Configuration;
|
||||||
|
import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import com.alibaba.datax.plugin.rdbms.reader.CommonRdbmsReader;
|
||||||
|
import com.alibaba.datax.plugin.rdbms.reader.Constant;
|
||||||
|
|
||||||
|
|
||||||
|
public class SybaseReader extends Reader {
|
||||||
|
|
||||||
|
private static final DataBaseType DATABASE_TYPE = DataBaseType.Oracle;
|
||||||
|
|
||||||
|
public static class Job extends Reader.Job {
|
||||||
|
private static final Logger LOG = LoggerFactory
|
||||||
|
.getLogger(SybaseReader.Job.class);
|
||||||
|
|
||||||
|
private Configuration originalConfig = null;
|
||||||
|
private CommonRdbmsReader.Job commonRdbmsReaderJob;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init() {
|
||||||
|
this.originalConfig = super.getPluginJobConf();
|
||||||
|
|
||||||
|
dealFetchSize(this.originalConfig);
|
||||||
|
|
||||||
|
this.commonRdbmsReaderJob = new CommonRdbmsReader.Job(
|
||||||
|
DATABASE_TYPE);
|
||||||
|
this.commonRdbmsReaderJob.init(this.originalConfig);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void preCheck(){
|
||||||
|
init();
|
||||||
|
this.commonRdbmsReaderJob.preCheck(this.originalConfig,DATABASE_TYPE);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Configuration> split(int adviceNumber) {
|
||||||
|
return this.commonRdbmsReaderJob.split(this.originalConfig,
|
||||||
|
adviceNumber);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void post() {
|
||||||
|
this.commonRdbmsReaderJob.post(this.originalConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void destroy() {
|
||||||
|
this.commonRdbmsReaderJob.destroy(this.originalConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void dealFetchSize(Configuration originalConfig) {
|
||||||
|
int fetchSize = originalConfig.getInt(
|
||||||
|
com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE,
|
||||||
|
Constants.DEFAULT_FETCH_SIZE);
|
||||||
|
if (fetchSize < 1) {
|
||||||
|
}
|
||||||
|
originalConfig.set(
|
||||||
|
com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE,
|
||||||
|
fetchSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class Task extends Reader.Task {
|
||||||
|
|
||||||
|
private Configuration readerSliceConfig;
|
||||||
|
private CommonRdbmsReader.Task commonRdbmsReaderTask;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init() {
|
||||||
|
this.readerSliceConfig = super.getPluginJobConf();
|
||||||
|
this.commonRdbmsReaderTask = new CommonRdbmsReader.Task(
|
||||||
|
DATABASE_TYPE ,super.getTaskGroupId(), super.getTaskId());
|
||||||
|
this.commonRdbmsReaderTask.init(this.readerSliceConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void startRead(RecordSender recordSender) {
|
||||||
|
int fetchSize = this.readerSliceConfig
|
||||||
|
.getInt(com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE);
|
||||||
|
|
||||||
|
this.commonRdbmsReaderTask.startRead(this.readerSliceConfig,
|
||||||
|
recordSender, super.getTaskPluginCollector(), fetchSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void post() {
|
||||||
|
this.commonRdbmsReaderTask.post(this.readerSliceConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void destroy() {
|
||||||
|
this.commonRdbmsReaderTask.destroy(this.readerSliceConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
6
sybasereader/src/main/resources/plugin.json
Executable file
6
sybasereader/src/main/resources/plugin.json
Executable file
@ -0,0 +1,6 @@
|
|||||||
|
{
|
||||||
|
"name": "sybasereader",
|
||||||
|
"class": "com.alibaba.datax.plugin.reader.sybasereader.SybaseReader",
|
||||||
|
"description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
|
||||||
|
"developer": "alibaba"
|
||||||
|
}
|
||||||
14
sybasereader/src/main/resources/plugin_job_template.json
Normal file
14
sybasereader/src/main/resources/plugin_job_template.json
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"name": "sybasereader",
|
||||||
|
"parameter": {
|
||||||
|
"username": "",
|
||||||
|
"password": "",
|
||||||
|
"column": [],
|
||||||
|
"connection": [
|
||||||
|
{
|
||||||
|
"table": [],
|
||||||
|
"jdbcUrl": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
19
sybasewriter/pom.xml
Normal file
19
sybasewriter/pom.xml
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<parent>
|
||||||
|
<artifactId>datax-all</artifactId>
|
||||||
|
<groupId>com.alibaba.datax</groupId>
|
||||||
|
<version>0.0.1-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<artifactId>sybasewriter</artifactId>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
</project>
|
||||||
Loading…
Reference in New Issue
Block a user