From fe0d41dd485412b9befcbdcf2f8535adc7e80712 Mon Sep 17 00:00:00 2001 From: mumu <“mumu081834@163.com”> Date: Fri, 25 Aug 2023 14:33:16 +0800 Subject: [PATCH 1/2] =?UTF-8?q?1.=20=E6=94=AF=E6=8C=81job=E4=B8=AD?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=A4=9A=E4=B8=AAcontent=EF=BC=9B2.=20String?= =?UTF-8?q?=E7=A9=BA=E5=AD=97=E7=AC=A6=E4=B8=B2=E8=BD=ACdate=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datax/common/element/ColumnCast.java | 11 +-- .../datax/common/element/DateColumn.java | 2 +- core/pom.xml | 5 ++ .../java/com/alibaba/datax/core/Engine.java | 13 +++ core/src/main/job/job_batch.json | 89 +++++++++++++++++++ 5 files changed, 114 insertions(+), 6 deletions(-) create mode 100755 core/src/main/job/job_batch.json diff --git a/common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java b/common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java index 85d62ecc..555202c9 100755 --- a/common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java +++ b/common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java @@ -3,6 +3,7 @@ package com.alibaba.datax.common.element; import com.alibaba.datax.common.exception.CommonErrorCode; import com.alibaba.datax.common.exception.DataXException; import com.alibaba.datax.common.util.Configuration; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateFormatUtils; import org.apache.commons.lang3.time.FastDateFormat; @@ -22,7 +23,7 @@ public final class ColumnCast { throws ParseException { return StringCast.asDate(column); } - + public static Date string2Date(final StringColumn column, String dateFormat) throws ParseException { return StringCast.asDate(column, dateFormat); @@ -90,7 +91,7 @@ class StringCast { } static Date asDate(final StringColumn column) throws ParseException { - if (null == column.asString()) { + if (StringUtils.isBlank(column.asString())) { return null; } @@ -120,7 +121,7 @@ class StringCast { } throw e; } - + static Date asDate(final StringColumn column, String dateFormat) throws ParseException { ParseException e; try { @@ -143,7 +144,7 @@ class StringCast { /** * 后续为了可维护性，可以考虑直接使用 apache 的DateFormatUtils. - * + * * 迟南已经修复了该问题，但是为了维护性，还是直接使用apache的内置函数 */ class DateCast { @@ -211,4 +212,4 @@ class BytesCast { return new String(column.asBytes(), encoding); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java b/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java index f688d163..9200dfb8 100755 --- a/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java +++ b/common/src/main/java/com/alibaba/datax/common/element/DateColumn.java @@ -132,4 +132,4 @@ public class DateColumn extends Column { public void setSubType(DateType subType) { this.subType = subType; } -} \ No newline at end of file +} diff --git a/core/pom.xml b/core/pom.xml index 7685001b..60baea64 100755 --- a/core/pom.xml +++ b/core/pom.xml @@ -97,6 +97,11 @@ groovy-all 2.1.9 + + commons-lang + commons-lang + 2.6 + diff --git a/core/src/main/java/com/alibaba/datax/core/Engine.java b/core/src/main/java/com/alibaba/datax/core/Engine.java index 4ba9fc18..477d4de5 100755 --- a/core/src/main/java/com/alibaba/datax/core/Engine.java +++ b/core/src/main/java/com/alibaba/datax/core/Engine.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.regex.Matcher; @@ -128,6 +129,15 @@ public class Engine { String jobIdString = cl.getOptionValue("jobid"); RUNTIME_MODE = cl.getOptionValue("mode"); + Configuration tmpConfiguration = ConfigParser.parse(jobPath); + List list = tmpConfiguration.getList("job.content"); + Iterator iterator = list.iterator(); + while (iterator.hasNext()) { + doEntry(jobPath, jobIdString, iterator.next()); + } + } + + private static void doEntry(String jobPath, String jobIdString, Object firstObj) { Configuration configuration = ConfigParser.parse(jobPath); // 绑定i18n信息 MessageSource.init(configuration); @@ -163,6 +173,8 @@ public class Engine { LOG.debug(configuration.toJSON()); + // mumu 2023/8/25 将任务添加至第一个(engine中默认取content中第一个任务进行处理，不太熟悉代码，而且改的话改动量太大，所以在入口处调整) + configuration.getList("job.content").add(0, firstObj); ConfigurationValidate.doValidate(configuration); Engine engine = new Engine(); engine.start(configuration); @@ -198,6 +210,7 @@ public class Engine { public static void main(String[] args) throws Exception { int exitCode = 0; try { + System.setProperty("datax.home", "/Users/mumu/workSpace/tools/datax"); Engine.entry(args); } catch (Throwable e) { exitCode = 1; diff --git a/core/src/main/job/job_batch.json b/core/src/main/job/job_batch.json new file mode 100755 index 00000000..64697ece --- /dev/null +++ b/core/src/main/job/job_batch.json @@ -0,0 +1,89 @@ +{ + "job": { + "setting": { + "speed": { + "channel":1 + }, + "errorLimit": { + "record": 0, + "percentage": 0.02 + } + }, + "content": [ + { + "reader": { + "name": "streamreader", + "parameter": { + "column" : [ + { + "value": "DataX", + "type": "string" + }, + { + "value": 19890604, + "type": "long" + }, + { + "value": "1989-06-04 00:00:00", + "type": "date" + }, + { + "value": true, + "type": "bool" + }, + { + "value": "test", + "type": "bytes" + } + ], + "sliceRecordCount": 10 + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "print": false, + "encoding": "UTF-8" + } + } + }, + { + "reader": { + "name": "streamreader", + "parameter": { + "column" : [ + { + "value": "DataX222", + "type": "string" + }, + { + "value": 19890604, + "type": "long" + }, + { + "value": "1989-06-04 00:00:00", + "type": "date" + }, + { + "value": true, + "type": "bool" + }, + { + "value": "test222", + "type": "bytes" + } + ], + "sliceRecordCount": 10 + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "print": true, + "encoding": "UTF-8" + } + } + } + ] + } +} From cdb6e706267054bdb599b70cc6021e4bf04573a5 Mon Sep 17 00:00:00 2001 From: mumu <“mumu081834@163.com”> Date: Mon, 28 Aug 2023 16:20:26 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/alibaba/datax/core/Engine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/alibaba/datax/core/Engine.java b/core/src/main/java/com/alibaba/datax/core/Engine.java index 477d4de5..c930b663 100755 --- a/core/src/main/java/com/alibaba/datax/core/Engine.java +++ b/core/src/main/java/com/alibaba/datax/core/Engine.java @@ -210,7 +210,7 @@ public class Engine { public static void main(String[] args) throws Exception { int exitCode = 0; try { - System.setProperty("datax.home", "/Users/mumu/workSpace/tools/datax"); +// System.setProperty("datax.home", "/Users/mumu/workSpace/tools/datax"); Engine.entry(args); } catch (Throwable e) { exitCode = 1;