mirror of
https://github.com/alibaba/DataX.git
synced 2025-05-02 00:53:34 +08:00
[feature][plugin][sybase] datax support sybase plugins. #1780
add reader plugin
This commit is contained in:
xiaopengm
parent
65a880b852
commit
e158e5845b
@ -25,7 +25,9 @@ public enum DataBaseType {
|
||||
Oscar("oscar", "com.oscar.Driver"),
|
||||
OceanBase("oceanbase", "com.alipay.oceanbase.jdbc.Driver"),
|
||||
StarRocks("starrocks", "com.mysql.jdbc.Driver"),
|
||||
Databend("databend", "com.databend.jdbc.DatabendDriver");
|
||||
Databend("databend", "com.databend.jdbc.DatabendDriver"),
|
||||
Sybase("sybase", "com.sybase.jdbc4.jdbc.SybDriver");
|
||||
|
||||
|
||||
private String typeName;
|
||||
private String driverClassName;
|
||||
|
||||
@ -233,12 +233,12 @@ SybaseReader插件实现了从Sybase读取数据。在底层实现上，SybaseRe
|
||||
|
||||
| DataX 内部类型| Sybase 数据类型 |
|
||||
| -------- | ----- |
|
||||
| Long |NUMBER,INTEGER,INT,SMALLINT|
|
||||
| Double |NUMERIC,DECIMAL,FLOAT,DOUBLE PRECISION,REAL|
|
||||
| String |LONG,CHAR,NCHAR,VARCHAR,VARCHAR2,NVARCHAR2,CLOB,NCLOB,CHARACTER,CHARACTER VARYING,CHAR VARYING,NATIONAL CHARACTER,NATIONAL CHAR,NATIONAL CHARACTER VARYING,NATIONAL CHAR VARYING,NCHAR VARYING |
|
||||
| Date |TIMESTAMP,DATE |
|
||||
| Boolean |bit, bool |
|
||||
| Bytes |BLOB,BFILE,RAW,LONG RAW |
|
||||
| Long |Tinyint,Smallint,Int,Money,Smallmoney|
|
||||
| Double |Float,Real,Numeric,Decimal|
|
||||
| String |Char,Varchar,Nchar,Nvarchar,Text|
|
||||
| Date |Timestamp,Datetime,Smalldatetime|
|
||||
| Boolean |bit, bool|
|
||||
| Bytes |Binary,Varbinary,Image|
|
||||
|
||||
|
||||
|
||||
|
||||
105
sybasereader/pom.xml
Normal file
105
sybasereader/pom.xml
Normal file
@ -0,0 +1,105 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>datax-all</artifactId>
|
||||
<groupId>com.alibaba.datax</groupId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>sybasereader</artifactId>
|
||||
<name>sybasereader</name>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.datax</groupId>
|
||||
<artifactId>datax-common</artifactId>
|
||||
<version>${datax-project-version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.alibaba.datax</groupId>
|
||||
<artifactId>plugin-rdbms-util</artifactId>
|
||||
<version>${datax-project-version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.oracle</groupId>
|
||||
<artifactId>ojdbc6</artifactId>
|
||||
<version>11.2.0.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.datax</groupId>
|
||||
<artifactId>datax-common</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.sybase.jconnect</groupId>
|
||||
<artifactId>jconn4</artifactId>
|
||||
<version>16.0</version>
|
||||
<scope>system</scope>
|
||||
<systemPath>${project.basedir}/libs/jconn4-16.0.jar</systemPath>
|
||||
</dependency>
|
||||
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<!-- compiler plugin -->
|
||||
<plugin>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>${jdk-version}</source>
|
||||
<target>${jdk-version}</target>
|
||||
<encoding>${project-sourceEncoding}</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<!-- assembly plugin -->
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<configuration>
|
||||
<descriptors>
|
||||
<descriptor>src/main/assembly/package.xml</descriptor>
|
||||
</descriptors>
|
||||
<finalName>datax</finalName>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>dwzip</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
</project>
|
||||
35
sybasereader/src/main/assembly/package.xml
Executable file
35
sybasereader/src/main/assembly/package.xml
Executable file
@ -0,0 +1,35 @@
|
||||
<assembly
|
||||
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||
<id></id>
|
||||
<formats>
|
||||
<format>dir</format>
|
||||
</formats>
|
||||
<includeBaseDirectory>false</includeBaseDirectory>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>src/main/resources</directory>
|
||||
<includes>
|
||||
<include>plugin.json</include>
|
||||
<include>plugin_job_template.json</include>
|
||||
</includes>
|
||||
<outputDirectory>plugin/reader/oraclereader</outputDirectory>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>target/</directory>
|
||||
<includes>
|
||||
<include>oraclereader-0.0.1-SNAPSHOT.jar</include>
|
||||
</includes>
|
||||
<outputDirectory>plugin/reader/oraclereader</outputDirectory>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
|
||||
<dependencySets>
|
||||
<dependencySet>
|
||||
<useProjectArtifact>false</useProjectArtifact>
|
||||
<outputDirectory>plugin/reader/oraclereader/libs</outputDirectory>
|
||||
<scope>runtime</scope>
|
||||
</dependencySet>
|
||||
</dependencySets>
|
||||
</assembly>
|
||||
@ -0,0 +1,7 @@
|
||||
package com.alibaba.datax.plugin.reader.sybasereader;
|
||||
|
||||
public class Constants {
|
||||
|
||||
public static final int DEFAULT_FETCH_SIZE = 1024;
|
||||
|
||||
}
|
||||
@ -0,0 +1,107 @@
|
||||
package com.alibaba.datax.plugin.reader.sybasereader;
|
||||
|
||||
import com.alibaba.datax.common.plugin.RecordSender;
|
||||
import com.alibaba.datax.common.spi.Reader;
|
||||
import com.alibaba.datax.common.util.Configuration;
|
||||
import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.List;
|
||||
import com.alibaba.datax.plugin.rdbms.reader.CommonRdbmsReader;
|
||||
import com.alibaba.datax.plugin.rdbms.reader.Constant;
|
||||
|
||||
|
||||
public class SybaseReader extends Reader {
|
||||
|
||||
private static final DataBaseType DATABASE_TYPE = DataBaseType.Oracle;
|
||||
|
||||
public static class Job extends Reader.Job {
|
||||
private static final Logger LOG = LoggerFactory
|
||||
.getLogger(SybaseReader.Job.class);
|
||||
|
||||
private Configuration originalConfig = null;
|
||||
private CommonRdbmsReader.Job commonRdbmsReaderJob;
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
this.originalConfig = super.getPluginJobConf();
|
||||
|
||||
dealFetchSize(this.originalConfig);
|
||||
|
||||
this.commonRdbmsReaderJob = new CommonRdbmsReader.Job(
|
||||
DATABASE_TYPE);
|
||||
this.commonRdbmsReaderJob.init(this.originalConfig);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void preCheck(){
|
||||
init();
|
||||
this.commonRdbmsReaderJob.preCheck(this.originalConfig,DATABASE_TYPE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Configuration> split(int adviceNumber) {
|
||||
return this.commonRdbmsReaderJob.split(this.originalConfig,
|
||||
adviceNumber);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void post() {
|
||||
this.commonRdbmsReaderJob.post(this.originalConfig);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
this.commonRdbmsReaderJob.destroy(this.originalConfig);
|
||||
}
|
||||
|
||||
private void dealFetchSize(Configuration originalConfig) {
|
||||
int fetchSize = originalConfig.getInt(
|
||||
com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE,
|
||||
Constants.DEFAULT_FETCH_SIZE);
|
||||
if (fetchSize < 1) {
|
||||
}
|
||||
originalConfig.set(
|
||||
com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE,
|
||||
fetchSize);
|
||||
}
|
||||
}
|
||||
|
||||
public static class Task extends Reader.Task {
|
||||
|
||||
private Configuration readerSliceConfig;
|
||||
private CommonRdbmsReader.Task commonRdbmsReaderTask;
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
this.readerSliceConfig = super.getPluginJobConf();
|
||||
this.commonRdbmsReaderTask = new CommonRdbmsReader.Task(
|
||||
DATABASE_TYPE ,super.getTaskGroupId(), super.getTaskId());
|
||||
this.commonRdbmsReaderTask.init(this.readerSliceConfig);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startRead(RecordSender recordSender) {
|
||||
int fetchSize = this.readerSliceConfig
|
||||
.getInt(com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE);
|
||||
|
||||
this.commonRdbmsReaderTask.startRead(this.readerSliceConfig,
|
||||
recordSender, super.getTaskPluginCollector(), fetchSize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void post() {
|
||||
this.commonRdbmsReaderTask.post(this.readerSliceConfig);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
this.commonRdbmsReaderTask.destroy(this.readerSliceConfig);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
6
sybasereader/src/main/resources/plugin.json
Executable file
6
sybasereader/src/main/resources/plugin.json
Executable file
@ -0,0 +1,6 @@
|
||||
{
|
||||
"name": "sybasereader",
|
||||
"class": "com.alibaba.datax.plugin.reader.sybasereader.SybaseReader",
|
||||
"description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
|
||||
"developer": "alibaba"
|
||||
}
|
||||
14
sybasereader/src/main/resources/plugin_job_template.json
Normal file
14
sybasereader/src/main/resources/plugin_job_template.json
Normal file
@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "sybasereader",
|
||||
"parameter": {
|
||||
"username": "",
|
||||
"password": "",
|
||||
"column": [],
|
||||
"connection": [
|
||||
{
|
||||
"table": [],
|
||||
"jdbcUrl": []
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
19
sybasewriter/pom.xml
Normal file
19
sybasewriter/pom.xml
Normal file
@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>datax-all</artifactId>
|
||||
<groupId>com.alibaba.datax</groupId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>sybasewriter</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
</properties>
|
||||
|
||||
</project>
|
||||
Loading…
Reference in New Issue
Block a user