mirror of
https://github.com/alibaba/DataX.git
synced 2025-05-02 05:11:42 +08:00
Merge cdb6e70626 into 0824b45c5e
This commit is contained in:
MUMU-FOREVER
GitHub
commit
700166cc30
@ -3,6 +3,7 @@ package com.alibaba.datax.common.element;
|
||||
import com.alibaba.datax.common.exception.CommonErrorCode;
|
||||
import com.alibaba.datax.common.exception.DataXException;
|
||||
import com.alibaba.datax.common.util.Configuration;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.time.DateFormatUtils;
|
||||
import org.apache.commons.lang3.time.FastDateFormat;
|
||||
|
||||
@ -22,7 +23,7 @@ public final class ColumnCast {
|
||||
throws ParseException {
|
||||
return StringCast.asDate(column);
|
||||
}
|
||||
|
||||
|
||||
public static Date string2Date(final StringColumn column, String dateFormat)
|
||||
throws ParseException {
|
||||
return StringCast.asDate(column, dateFormat);
|
||||
@ -90,7 +91,7 @@ class StringCast {
|
||||
}
|
||||
|
||||
static Date asDate(final StringColumn column) throws ParseException {
|
||||
if (null == column.asString()) {
|
||||
if (StringUtils.isBlank(column.asString())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -120,7 +121,7 @@ class StringCast {
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
|
||||
static Date asDate(final StringColumn column, String dateFormat) throws ParseException {
|
||||
ParseException e;
|
||||
try {
|
||||
@ -143,7 +144,7 @@ class StringCast {
|
||||
|
||||
/**
|
||||
* 后续为了可维护性，可以考虑直接使用 apache 的DateFormatUtils.
|
||||
*
|
||||
*
|
||||
* 迟南已经修复了该问题，但是为了维护性，还是直接使用apache的内置函数
|
||||
*/
|
||||
class DateCast {
|
||||
@ -211,4 +212,4 @@ class BytesCast {
|
||||
|
||||
return new String(column.asBytes(), encoding);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -169,4 +169,4 @@ public class DateColumn extends Column {
|
||||
public void setSubType(DateType subType) {
|
||||
this.subType = subType;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -97,6 +97,11 @@
|
||||
<artifactId>groovy-all</artifactId>
|
||||
<version>2.1.9</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-lang</groupId>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
<version>2.6</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
@ -23,6 +23,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
@ -128,6 +129,15 @@ public class Engine {
|
||||
String jobIdString = cl.getOptionValue("jobid");
|
||||
RUNTIME_MODE = cl.getOptionValue("mode");
|
||||
|
||||
Configuration tmpConfiguration = ConfigParser.parse(jobPath);
|
||||
List<Object> list = tmpConfiguration.getList("job.content");
|
||||
Iterator<Object> iterator = list.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
doEntry(jobPath, jobIdString, iterator.next());
|
||||
}
|
||||
}
|
||||
|
||||
private static void doEntry(String jobPath, String jobIdString, Object firstObj) {
|
||||
Configuration configuration = ConfigParser.parse(jobPath);
|
||||
// 绑定i18n信息
|
||||
MessageSource.init(configuration);
|
||||
@ -163,6 +173,8 @@ public class Engine {
|
||||
|
||||
LOG.debug(configuration.toJSON());
|
||||
|
||||
// mumu 2023/8/25 将任务添加至第一个(engine中默认取content中第一个任务进行处理，不太熟悉代码，而且改的话改动量太大，所以在入口处调整)
|
||||
configuration.getList("job.content").add(0, firstObj);
|
||||
ConfigurationValidate.doValidate(configuration);
|
||||
Engine engine = new Engine();
|
||||
engine.start(configuration);
|
||||
@ -198,6 +210,7 @@ public class Engine {
|
||||
public static void main(String[] args) throws Exception {
|
||||
int exitCode = 0;
|
||||
try {
|
||||
// System.setProperty("datax.home", "/Users/mumu/workSpace/tools/datax");
|
||||
Engine.entry(args);
|
||||
} catch (Throwable e) {
|
||||
exitCode = 1;
|
||||
|
||||
89
core/src/main/job/job_batch.json
Executable file
89
core/src/main/job/job_batch.json
Executable file
@ -0,0 +1,89 @@
|
||||
{
|
||||
"job": {
|
||||
"setting": {
|
||||
"speed": {
|
||||
"channel":1
|
||||
},
|
||||
"errorLimit": {
|
||||
"record": 0,
|
||||
"percentage": 0.02
|
||||
}
|
||||
},
|
||||
"content": [
|
||||
{
|
||||
"reader": {
|
||||
"name": "streamreader",
|
||||
"parameter": {
|
||||
"column" : [
|
||||
{
|
||||
"value": "DataX",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"value": 19890604,
|
||||
"type": "long"
|
||||
},
|
||||
{
|
||||
"value": "1989-06-04 00:00:00",
|
||||
"type": "date"
|
||||
},
|
||||
{
|
||||
"value": true,
|
||||
"type": "bool"
|
||||
},
|
||||
{
|
||||
"value": "test",
|
||||
"type": "bytes"
|
||||
}
|
||||
],
|
||||
"sliceRecordCount": 10
|
||||
}
|
||||
},
|
||||
"writer": {
|
||||
"name": "streamwriter",
|
||||
"parameter": {
|
||||
"print": false,
|
||||
"encoding": "UTF-8"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"reader": {
|
||||
"name": "streamreader",
|
||||
"parameter": {
|
||||
"column" : [
|
||||
{
|
||||
"value": "DataX222",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"value": 19890604,
|
||||
"type": "long"
|
||||
},
|
||||
{
|
||||
"value": "1989-06-04 00:00:00",
|
||||
"type": "date"
|
||||
},
|
||||
{
|
||||
"value": true,
|
||||
"type": "bool"
|
||||
},
|
||||
{
|
||||
"value": "test222",
|
||||
"type": "bytes"
|
||||
}
|
||||
],
|
||||
"sliceRecordCount": 10
|
||||
}
|
||||
},
|
||||
"writer": {
|
||||
"name": "streamwriter",
|
||||
"parameter": {
|
||||
"print": true,
|
||||
"encoding": "UTF-8"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user