github/DataX
5
0
Fork 0
mirror of https://github.com/alibaba/DataX.git synced 2025-05-02 11:11:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge cdb6e70626 into 0824b45c5e

Browse Source
This commit is contained in:
MUMU-FOREVER 2025-04-10 16:20:18 +08:00 committed by GitHub
commit 700166cc30
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 114 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java
View File

@ -3,6 +3,7 @@ package com.alibaba.datax.common.element;
import com.alibaba.datax.common.exception.CommonErrorCode;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.lang3.time.FastDateFormat;
@ -90,7 +91,7 @@ class StringCast {
}
static Date asDate(final StringColumn column) throws ParseException {
if (null == column.asString()) {
if (StringUtils.isBlank(column.asString())) {
return null;
}

5
core/pom.xml
View File

@ -97,6 +97,11 @@
<artifactId>groovy-all</artifactId>
<version>2.1.9</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
</dependencies>
<build>

13
core/src/main/java/com/alibaba/datax/core/Engine.java
View File

@ -23,6 +23,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
@ -128,6 +129,15 @@ public class Engine {
String jobIdString = cl.getOptionValue("jobid");
RUNTIME_MODE = cl.getOptionValue("mode");
Configuration tmpConfiguration = ConfigParser.parse(jobPath);
List<Object> list = tmpConfiguration.getList("job.content");
Iterator<Object> iterator = list.iterator();
while (iterator.hasNext()) {
doEntry(jobPath, jobIdString, iterator.next());
}
}
private static void doEntry(String jobPath, String jobIdString, Object firstObj) {
Configuration configuration = ConfigParser.parse(jobPath);
// 绑定i18n信息
MessageSource.init(configuration);
@ -163,6 +173,8 @@ public class Engine {
LOG.debug(configuration.toJSON());
// mumu 2023/8/25 将任务添加至第一个(engine中默认取content中第一个任务进行处理，不太熟悉代码，而且改的话改动量太大，所以在入口处调整)
configuration.getList("job.content").add(0, firstObj);
ConfigurationValidate.doValidate(configuration);
Engine engine = new Engine();
engine.start(configuration);
@ -198,6 +210,7 @@ public class Engine {
public static void main(String[] args) throws Exception {
int exitCode = 0;
try {
// System.setProperty("datax.home", "/Users/mumu/workSpace/tools/datax");
Engine.entry(args);
} catch (Throwable e) {
exitCode = 1;

89
core/src/main/job/job_batch.json Executable file
View File

@ -0,0 +1,89 @@
{
"job": {
"setting": {
"speed": {
"channel":1
},
"errorLimit": {
"record": 0,
"percentage": 0.02
}
},
"content": [
{
"reader": {
"name": "streamreader",
"parameter": {
"column" : [
{
"value": "DataX",
"type": "string"
},
{
"value": 19890604,
"type": "long"
},
{
"value": "1989-06-04 00:00:00",
"type": "date"
},
{
"value": true,
"type": "bool"
},
{
"value": "test",
"type": "bytes"
}
],
"sliceRecordCount": 10
}
},
"writer": {
"name": "streamwriter",
"parameter": {
"print": false,
"encoding": "UTF-8"
}
}
},
{
"reader": {
"name": "streamreader",
"parameter": {
"column" : [
{
"value": "DataX222",
"type": "string"
},
{
"value": 19890604,
"type": "long"
},
{
"value": "1989-06-04 00:00:00",
"type": "date"
},
{
"value": true,
"type": "bool"
},
{
"value": "test222",
"type": "bytes"
}
],
"sliceRecordCount": 10
}
},
"writer": {
"name": "streamwriter",
"parameter": {
"print": true,
"encoding": "UTF-8"
}
}
}
]
}
}