",
- action="store", dest="writer",type="string",
+ action="store", dest="writer", type="string",
help='View job config[writer] template, eg: mysqlwriter,streamwriter')
parser.add_option_group(prodEnvOptionGroup)
@@ -108,45 +117,50 @@ def getOptionParser():
parser.add_option_group(devEnvOptionGroup)
return parser
+
def generateJobConfigTemplate(reader, writer):
- readerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n" % (reader,reader,reader)
- writerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n " % (writer,writer,writer)
- print readerRef
- print writerRef
+ readerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n" % (
+ reader, reader, reader)
+ writerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n " % (
+ writer, writer, writer)
+ print(readerRef)
+ print(writerRef)
jobGuid = 'Please save the following configuration as a json file and use\n python {DATAX_HOME}/bin/datax.py {JSON_FILE_NAME}.json \nto run the job.\n'
- print jobGuid
- jobTemplate={
- "job": {
- "setting": {
- "speed": {
- "channel": ""
- }
- },
- "content": [
- {
- "reader": {},
- "writer": {}
- }
- ]
- }
+ print(jobGuid)
+ jobTemplate = {
+ "job": {
+ "setting": {
+ "speed": {
+ "channel": ""
+ }
+ },
+ "content": [
+ {
+ "reader": {},
+ "writer": {}
+ }
+ ]
+ }
}
- readerTemplatePath = "%s/plugin/reader/%s/plugin_job_template.json" % (DATAX_HOME,reader)
- writerTemplatePath = "%s/plugin/writer/%s/plugin_job_template.json" % (DATAX_HOME,writer)
+ readerTemplatePath = "%s/plugin/reader/%s/plugin_job_template.json" % (DATAX_HOME, reader)
+ writerTemplatePath = "%s/plugin/writer/%s/plugin_job_template.json" % (DATAX_HOME, writer)
try:
- readerPar = readPluginTemplate(readerTemplatePath);
- except Exception, e:
- print "Read reader[%s] template error: can\'t find file %s" % (reader,readerTemplatePath)
+ readerPar = readPluginTemplate(readerTemplatePath)
+ except:
+ print("Read reader[%s] template error: can\'t find file %s" % (reader, readerTemplatePath))
try:
- writerPar = readPluginTemplate(writerTemplatePath);
- except Exception, e:
- print "Read writer[%s] template error: : can\'t find file %s" % (writer,writerTemplatePath)
- jobTemplate['job']['content'][0]['reader'] = readerPar;
- jobTemplate['job']['content'][0]['writer'] = writerPar;
- print json.dumps(jobTemplate, indent=4, sort_keys=True)
+ writerPar = readPluginTemplate(writerTemplatePath)
+ except:
+ print("Read writer[%s] template error: : can\'t find file %s" % (writer, writerTemplatePath))
+ jobTemplate['job']['content'][0]['reader'] = readerPar
+ jobTemplate['job']['content'][0]['writer'] = writerPar
+ print(json.dumps(jobTemplate, indent=4, sort_keys=True))
+
def readPluginTemplate(plugin):
with open(plugin, 'r') as f:
- return json.load(f)
+ return json.load(f)
+
def isUrl(path):
if not path:
@@ -168,7 +182,7 @@ def buildStartCommand(options, args):
if options.remoteDebug:
tempJVMCommand = tempJVMCommand + " " + REMOTE_DEBUG_CONFIG
- print 'local ip: ', getLocalIp()
+ print('local ip: ', getLocalIp())
if options.loglevel:
tempJVMCommand = tempJVMCommand + " " + ("-Dloglevel=%s" % (options.loglevel))
@@ -198,11 +212,11 @@ def buildStartCommand(options, args):
def printCopyright():
- print '''
+ print('''
DataX (%s), From Alibaba !
Copyright (C) 2010-2017, Alibaba Group. All Rights Reserved.
-''' % DATAX_VERSION
+''' % DATAX_VERSION)
sys.stdout.flush()
@@ -211,7 +225,7 @@ if __name__ == "__main__":
parser = getOptionParser()
options, args = parser.parse_args(sys.argv[1:])
if options.reader is not None and options.writer is not None:
- generateJobConfigTemplate(options.reader,options.writer)
+ generateJobConfigTemplate(options.reader, options.writer)
sys.exit(RET_STATE['OK'])
if len(args) != 1:
parser.print_help()
diff --git a/core/src/main/java/com/alibaba/datax/core/Engine.java b/core/src/main/java/com/alibaba/datax/core/Engine.java
index f80d792f..4ba9fc18 100755
--- a/core/src/main/java/com/alibaba/datax/core/Engine.java
+++ b/core/src/main/java/com/alibaba/datax/core/Engine.java
@@ -6,6 +6,7 @@ import com.alibaba.datax.common.spi.ErrorCode;
import com.alibaba.datax.common.statistics.PerfTrace;
import com.alibaba.datax.common.statistics.VMInfo;
import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.util.MessageSource;
import com.alibaba.datax.core.job.JobContainer;
import com.alibaba.datax.core.taskgroup.TaskGroupContainer;
import com.alibaba.datax.core.util.ConfigParser;
@@ -73,21 +74,14 @@ public class Engine {
boolean traceEnable = allConf.getBool(CoreConstant.DATAX_CORE_CONTAINER_TRACE_ENABLE, true);
boolean perfReportEnable = allConf.getBool(CoreConstant.DATAX_CORE_REPORT_DATAX_PERFLOG, true);
- //standlone模式的datax shell任务不进行汇报
+ //standalone模式的 datax shell任务不进行汇报
if(instanceId == -1){
perfReportEnable = false;
}
- int priority = 0;
- try {
- priority = Integer.parseInt(System.getenv("SKYNET_PRIORITY"));
- }catch (NumberFormatException e){
- LOG.warn("prioriy set to 0, because NumberFormatException, the value is: "+System.getProperty("PROIORY"));
- }
-
Configuration jobInfoConfig = allConf.getConfiguration(CoreConstant.DATAX_JOB_JOBINFO);
//初始化PerfTrace
- PerfTrace perfTrace = PerfTrace.getInstance(isJob, instanceId, taskGroupId, priority, traceEnable);
+ PerfTrace perfTrace = PerfTrace.getInstance(isJob, instanceId, taskGroupId, traceEnable);
perfTrace.setJobInfo(jobInfoConfig,perfReportEnable,channelNumber);
container.start();
@@ -135,6 +129,9 @@ public class Engine {
RUNTIME_MODE = cl.getOptionValue("mode");
Configuration configuration = ConfigParser.parse(jobPath);
+ // 绑定i18n信息
+ MessageSource.init(configuration);
+ MessageSource.reloadResourceBundle(Configuration.class);
long jobId;
if (!"-1".equalsIgnoreCase(jobIdString)) {
diff --git a/core/src/main/java/com/alibaba/datax/core/LocalStrings.properties b/core/src/main/java/com/alibaba/datax/core/LocalStrings.properties
new file mode 100644
index 00000000..97d46f07
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/LocalStrings.properties
@@ -0,0 +1,5 @@
+very_like_yixiao=\u4e00{0}\u4e8c{1}\u4e09
+
+engine.1=\u975e standalone \u6a21\u5f0f\u5fc5\u987b\u5728 URL \u4e2d\u63d0\u4f9b\u6709\u6548\u7684 jobId.
+engine.2=\n\n\u7ecfDataX\u667a\u80fd\u5206\u6790,\u8be5\u4efb\u52a1\u6700\u53ef\u80fd\u7684\u9519\u8bef\u539f\u56e0\u662f:\n{0}
+
diff --git a/core/src/main/java/com/alibaba/datax/core/LocalStrings_en_US.properties b/core/src/main/java/com/alibaba/datax/core/LocalStrings_en_US.properties
new file mode 100644
index 00000000..7ff93838
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/LocalStrings_en_US.properties
@@ -0,0 +1,5 @@
+very_like_yixiao=1{0}2{1}3
+
+engine.1=A valid job ID must be provided in the URL for the non-standalone mode.
+engine.2=\n\nThrough the intelligent analysis by DataX, the most likely error reason of this task is: \n{0}
+
diff --git a/core/src/main/java/com/alibaba/datax/core/LocalStrings_ja_JP.properties b/core/src/main/java/com/alibaba/datax/core/LocalStrings_ja_JP.properties
new file mode 100644
index 00000000..dfbad970
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/LocalStrings_ja_JP.properties
@@ -0,0 +1,5 @@
+very_like_yixiao=1{0}2{1}3
+
+engine.1=\u975e standalone \u6a21\u5f0f\u5fc5\u987b\u5728 URL \u4e2d\u63d0\u4f9b\u6709\u6548\u7684 jobId.
+engine.2=\n\n\u7ecfDataX\u667a\u80fd\u5206\u6790,\u8be5\u4efb\u52a1\u6700\u53ef\u80fd\u7684\u9519\u8bef\u539f\u56e0\u662f:\n{0}
+
diff --git a/core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_CN.properties b/core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_CN.properties
new file mode 100644
index 00000000..97d46f07
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_CN.properties
@@ -0,0 +1,5 @@
+very_like_yixiao=\u4e00{0}\u4e8c{1}\u4e09
+
+engine.1=\u975e standalone \u6a21\u5f0f\u5fc5\u987b\u5728 URL \u4e2d\u63d0\u4f9b\u6709\u6548\u7684 jobId.
+engine.2=\n\n\u7ecfDataX\u667a\u80fd\u5206\u6790,\u8be5\u4efb\u52a1\u6700\u53ef\u80fd\u7684\u9519\u8bef\u539f\u56e0\u662f:\n{0}
+
diff --git a/core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_HK.properties b/core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_HK.properties
new file mode 100644
index 00000000..2587e0ab
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_HK.properties
@@ -0,0 +1,10 @@
+very_like_yixiao=\u4e00{0}\u4e8c{1}\u4e09
+
+engine.1=\u975e standalone \u6a21\u5f0f\u5fc5\u987b\u5728 URL \u4e2d\u63d0\u4f9b\u6709\u6548\u7684 jobId.
+engine.2=\n\n\u7ecfDataX\u667a\u80fd\u5206\u6790,\u8be5\u4efb\u52a1\u6700\u53ef\u80fd\u7684\u9519\u8bef\u539f\u56e0\u662f:\n{0}
+
+very_like_yixiao=一{0}二{1}三
+
+engine.1=非 standalone 模式必須在 URL 中提供有效的 jobId.
+engine.2=\n\n經DataX智能分析,該任務最可能的錯誤原因是:\n{0}
+
diff --git a/core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_TW.properties b/core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_TW.properties
new file mode 100644
index 00000000..2587e0ab
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_TW.properties
@@ -0,0 +1,10 @@
+very_like_yixiao=\u4e00{0}\u4e8c{1}\u4e09
+
+engine.1=\u975e standalone \u6a21\u5f0f\u5fc5\u987b\u5728 URL \u4e2d\u63d0\u4f9b\u6709\u6548\u7684 jobId.
+engine.2=\n\n\u7ecfDataX\u667a\u80fd\u5206\u6790,\u8be5\u4efb\u52a1\u6700\u53ef\u80fd\u7684\u9519\u8bef\u539f\u56e0\u662f:\n{0}
+
+very_like_yixiao=一{0}二{1}三
+
+engine.1=非 standalone 模式必須在 URL 中提供有效的 jobId.
+engine.2=\n\n經DataX智能分析,該任務最可能的錯誤原因是:\n{0}
+
diff --git a/core/src/main/java/com/alibaba/datax/core/container/util/JobAssignUtil.java b/core/src/main/java/com/alibaba/datax/core/container/util/JobAssignUtil.java
index 31ba60a4..cbd0d2a1 100755
--- a/core/src/main/java/com/alibaba/datax/core/container/util/JobAssignUtil.java
+++ b/core/src/main/java/com/alibaba/datax/core/container/util/JobAssignUtil.java
@@ -114,7 +114,7 @@ public final class JobAssignUtil {
* 需要实现的效果通过例子来说是：
*
* a 库上有表：0, 1, 2
- * a 库上有表：3, 4
+ * b 库上有表：3, 4
* c 库上有表：5, 6, 7
*
* 如果有 4个 taskGroup
diff --git a/core/src/main/java/com/alibaba/datax/core/job/JobContainer.java b/core/src/main/java/com/alibaba/datax/core/job/JobContainer.java
index 26b2989f..49f5a0a1 100755
--- a/core/src/main/java/com/alibaba/datax/core/job/JobContainer.java
+++ b/core/src/main/java/com/alibaba/datax/core/job/JobContainer.java
@@ -27,7 +27,7 @@ import com.alibaba.datax.core.util.container.ClassLoaderSwapper;
import com.alibaba.datax.core.util.container.CoreConstant;
import com.alibaba.datax.core.util.container.LoadUtil;
import com.alibaba.datax.dataxservice.face.domain.enums.ExecuteMode;
-import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson2.JSON;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.Validate;
import org.slf4j.Logger;
diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/communication/CommunicationTool.java b/core/src/main/java/com/alibaba/datax/core/statistics/communication/CommunicationTool.java
index 51a601ae..1815ea02 100755
--- a/core/src/main/java/com/alibaba/datax/core/statistics/communication/CommunicationTool.java
+++ b/core/src/main/java/com/alibaba/datax/core/statistics/communication/CommunicationTool.java
@@ -2,7 +2,7 @@ package com.alibaba.datax.core.statistics.communication;
import com.alibaba.datax.common.statistics.PerfTrace;
import com.alibaba.datax.common.util.StrUtil;
-import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson2.JSON;
import org.apache.commons.lang.Validate;
import java.text.DecimalFormat;
diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/StdoutPluginCollector.java b/core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/StdoutPluginCollector.java
index 8b2a8378..d88ad0a8 100755
--- a/core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/StdoutPluginCollector.java
+++ b/core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/StdoutPluginCollector.java
@@ -6,7 +6,7 @@ import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.core.statistics.communication.Communication;
import com.alibaba.datax.core.util.container.CoreConstant;
import com.alibaba.datax.core.statistics.plugin.task.util.DirtyRecord;
-import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson2.JSON;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
diff --git a/core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/util/DirtyRecord.java b/core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/util/DirtyRecord.java
index fdc5d821..caa4cb5b 100755
--- a/core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/util/DirtyRecord.java
+++ b/core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/util/DirtyRecord.java
@@ -4,22 +4,25 @@ import com.alibaba.datax.common.element.Column;
import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.core.util.FrameworkErrorCode;
-import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson2.JSON;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.Map;
public class DirtyRecord implements Record {
private List columns = new ArrayList();
+ private Map meta;
public static DirtyRecord asDirtyRecord(final Record record) {
DirtyRecord result = new DirtyRecord();
for (int i = 0; i < record.getColumnNumber(); i++) {
result.addColumn(record.getColumn(i));
}
+ result.setMeta(record.getMeta());
return result;
}
@@ -65,6 +68,16 @@ public class DirtyRecord implements Record {
"该方法不支持!");
}
+ @Override
+ public void setMeta(Map meta) {
+ this.meta = meta;
+ }
+
+ @Override
+ public Map getMeta() {
+ return this.meta;
+ }
+
public List getColumns() {
return columns;
}
@@ -119,6 +132,12 @@ class DirtyColumn extends Column {
throw DataXException.asDataXException(FrameworkErrorCode.RUNTIME_ERROR,
"该方法不支持!");
}
+
+ @Override
+ public Date asDate(String dateFormat) {
+ throw DataXException.asDataXException(FrameworkErrorCode.RUNTIME_ERROR,
+ "该方法不支持!");
+ }
@Override
public byte[] asBytes() {
diff --git a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java
index c30c94d9..b4b45695 100755
--- a/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java
+++ b/core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java
@@ -27,7 +27,7 @@ import com.alibaba.datax.core.util.TransformerUtil;
import com.alibaba.datax.core.util.container.CoreConstant;
import com.alibaba.datax.core.util.container.LoadUtil;
import com.alibaba.datax.dataxservice.face.domain.enums.State;
-import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson2.JSON;
import org.apache.commons.lang3.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/alibaba/datax/core/transport/record/DefaultRecord.java b/core/src/main/java/com/alibaba/datax/core/transport/record/DefaultRecord.java
index 2598bc8c..1dfa02e8 100755
--- a/core/src/main/java/com/alibaba/datax/core/transport/record/DefaultRecord.java
+++ b/core/src/main/java/com/alibaba/datax/core/transport/record/DefaultRecord.java
@@ -5,7 +5,7 @@ import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.core.util.ClassSize;
import com.alibaba.datax.core.util.FrameworkErrorCode;
-import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson2.JSON;
import java.util.ArrayList;
import java.util.HashMap;
@@ -27,6 +27,8 @@ public class DefaultRecord implements Record {
// 首先是Record本身需要的内存
private int memorySize = ClassSize.DefaultRecordHead;
+ private Map meta;
+
public DefaultRecord() {
this.columns = new ArrayList(RECORD_AVERGAE_COLUMN_NUMBER);
}
@@ -83,6 +85,16 @@ public class DefaultRecord implements Record {
return memorySize;
}
+ @Override
+ public void setMeta(Map meta) {
+ this.meta = meta;
+ }
+
+ @Override
+ public Map getMeta() {
+ return this.meta;
+ }
+
private void decrByteSize(final Column column) {
if (null == column) {
return;
diff --git a/core/src/main/java/com/alibaba/datax/core/transport/record/TerminateRecord.java b/core/src/main/java/com/alibaba/datax/core/transport/record/TerminateRecord.java
index 928609ab..7cb1cff1 100755
--- a/core/src/main/java/com/alibaba/datax/core/transport/record/TerminateRecord.java
+++ b/core/src/main/java/com/alibaba/datax/core/transport/record/TerminateRecord.java
@@ -3,6 +3,8 @@ package com.alibaba.datax.core.transport.record;
import com.alibaba.datax.common.element.Column;
import com.alibaba.datax.common.element.Record;
+import java.util.Map;
+
/**
* 作为标示 生产者已经完成生产的标志
*
@@ -41,6 +43,16 @@ public class TerminateRecord implements Record {
return 0;
}
+ @Override
+ public void setMeta(Map meta) {
+
+ }
+
+ @Override
+ public Map getMeta() {
+ return null;
+ }
+
@Override
public void setColumn(int i, Column column) {
return;
diff --git a/core/src/main/java/com/alibaba/datax/core/transport/transformer/DigestTransformer.java b/core/src/main/java/com/alibaba/datax/core/transport/transformer/DigestTransformer.java
new file mode 100644
index 00000000..d2bf1431
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/transport/transformer/DigestTransformer.java
@@ -0,0 +1,87 @@
+package com.alibaba.datax.core.transport.transformer;
+
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.element.StringColumn;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.transformer.Transformer;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Arrays;
+
+/**
+ * no comments.
+ *
+ * @author XuDaojie
+ * @since 2021-08-16
+ */
+public class DigestTransformer extends Transformer {
+
+ private static final String MD5 = "md5";
+ private static final String SHA1 = "sha1";
+ private static final String TO_UPPER_CASE = "toUpperCase";
+ private static final String TO_LOWER_CASE = "toLowerCase";
+
+ public DigestTransformer() {
+ setTransformerName("dx_digest");
+ }
+
+ @Override
+ public Record evaluate(Record record, Object... paras) {
+
+ int columnIndex;
+ String type;
+ String charType;
+
+ try {
+ if (paras.length != 3) {
+ throw new RuntimeException("dx_digest paras length must be 3");
+ }
+
+ columnIndex = (Integer) paras[0];
+ type = (String) paras[1];
+ charType = (String) paras[2];
+
+ if (!StringUtils.equalsIgnoreCase(MD5, type) && !StringUtils.equalsIgnoreCase(SHA1, type)) {
+ throw new RuntimeException("dx_digest paras index 1 must be md5 or sha1");
+ }
+ if (!StringUtils.equalsIgnoreCase(TO_UPPER_CASE, charType) && !StringUtils.equalsIgnoreCase(TO_LOWER_CASE, charType)) {
+ throw new RuntimeException("dx_digest paras index 2 must be toUpperCase or toLowerCase");
+ }
+ } catch (Exception e) {
+ throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, "paras:" + Arrays.asList(paras) + " => " + e.getMessage());
+ }
+
+ Column column = record.getColumn(columnIndex);
+
+ try {
+ String oriValue = column.asString();
+
+ // 如果字段为空，作为空字符串处理
+ if (oriValue == null) {
+ oriValue = "";
+ }
+ String newValue;
+ if (MD5.equals(type)) {
+ newValue = DigestUtils.md5Hex(oriValue);
+ } else {
+ newValue = DigestUtils.sha1Hex(oriValue);
+ }
+
+ if (TO_UPPER_CASE.equals(charType)) {
+ newValue = newValue.toUpperCase();
+ } else {
+ newValue = newValue.toLowerCase();
+ }
+
+ record.setColumn(columnIndex, new StringColumn(newValue));
+
+ } catch (Exception e) {
+ throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e);
+ }
+ return record;
+ }
+
+}
diff --git a/core/src/main/java/com/alibaba/datax/core/transport/transformer/FilterTransformer.java b/core/src/main/java/com/alibaba/datax/core/transport/transformer/FilterTransformer.java
index 8f6492fa..a3251715 100644
--- a/core/src/main/java/com/alibaba/datax/core/transport/transformer/FilterTransformer.java
+++ b/core/src/main/java/com/alibaba/datax/core/transport/transformer/FilterTransformer.java
@@ -61,7 +61,7 @@ public class FilterTransformer extends Transformer {
} else if (code.equalsIgnoreCase("<=")) {
return doLess(record, value, column, true);
} else {
- throw new RuntimeException("dx_filter can't suport code:" + code);
+ throw new RuntimeException("dx_filter can't support code:" + code);
}
} catch (Exception e) {
throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e);
diff --git a/core/src/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformerStaticUtil.java b/core/src/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformerStaticUtil.java
index 4c872993..487a8be8 100644
--- a/core/src/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformerStaticUtil.java
+++ b/core/src/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformerStaticUtil.java
@@ -1,10 +1,18 @@
package com.alibaba.datax.core.transport.transformer;
+import org.apache.commons.codec.digest.DigestUtils;
+
/**
* GroovyTransformer的帮助类，供groovy代码使用，必须全是static的方法
* Created by liqiang on 16/3/4.
*/
public class GroovyTransformerStaticUtil {
+ public static String md5(final String data) {
+ return DigestUtils.md5Hex(data);
+ }
+ public static String sha1(final String data) {
+ return DigestUtils.sha1Hex(data);
+ }
}
diff --git a/core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerRegistry.java b/core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerRegistry.java
index 96a0d988..3c625153 100644
--- a/core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerRegistry.java
+++ b/core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerRegistry.java
@@ -36,6 +36,7 @@ public class TransformerRegistry {
registTransformer(new ReplaceTransformer());
registTransformer(new FilterTransformer());
registTransformer(new GroovyTransformer());
+ registTransformer(new DigestTransformer());
}
public static void loadTransformerFromLocalStorage() {
diff --git a/core/src/main/java/com/alibaba/datax/core/util/LocalStrings.properties b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings.properties
new file mode 100644
index 00000000..a90f7829
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings.properties
@@ -0,0 +1,58 @@
+configparser.1=\u63D2\u4EF6[{0},{1}]\u52A0\u8F7D\u5931\u8D25\uFF0C1s\u540E\u91CD\u8BD5... Exception:{2}
+configparser.2=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.3=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.4=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.5=\u63D2\u4EF6\u52A0\u8F7D\u5931\u8D25\uFF0C\u672A\u5B8C\u6210\u6307\u5B9A\u63D2\u4EF6\u52A0\u8F7D:{0}
+configparser.6=\u63D2\u4EF6\u52A0\u8F7D\u5931\u8D25,\u5B58\u5728\u91CD\u590D\u63D2\u4EF6:{0}
+
+dataxserviceutil.1=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38NoSuchAlgorithmException, [{0}]
+dataxserviceutil.2=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38InvalidKeyException, [{0}]
+dataxserviceutil.3=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38UnsupportedEncodingException, [{0}]
+
+errorrecordchecker.1=\u810F\u6570\u636E\u767E\u5206\u6BD4\u9650\u5236\u5E94\u8BE5\u5728[0.0, 1.0]\u4E4B\u95F4
+errorrecordchecker.2=\u810F\u6570\u636E\u6761\u6570\u73B0\u5728\u5E94\u8BE5\u4E3A\u975E\u8D1F\u6574\u6570
+errorrecordchecker.3=\u810F\u6570\u636E\u6761\u6570\u68C0\u67E5\u4E0D\u901A\u8FC7\uFF0C\u9650\u5236\u662F[{0}]\u6761\uFF0C\u4F46\u5B9E\u9645\u4E0A\u6355\u83B7\u4E86[{1}]\u6761.
+errorrecordchecker.4=\u810F\u6570\u636E\u767E\u5206\u6BD4\u68C0\u67E5\u4E0D\u901A\u8FC7\uFF0C\u9650\u5236\u662F[{0}]\uFF0C\u4F46\u5B9E\u9645\u4E0A\u6355\u83B7\u5230[{1}].
+
+
+errorcode.install_error=DataX\u5F15\u64CE\u5B89\u88C5\u9519\u8BEF, \u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.argument_error=DataX\u5F15\u64CE\u8FD0\u884C\u9519\u8BEF\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8E\u5185\u90E8\u7F16\u7A0B\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3 .
+errorcode.runtime_error=DataX\u5F15\u64CE\u8FD0\u884C\u8FC7\u7A0B\u51FA\u9519\uFF0C\u5177\u4F53\u539F\u56E0\u8BF7\u53C2\u770BDataX\u8FD0\u884C\u7ED3\u675F\u65F6\u7684\u9519\u8BEF\u8BCA\u65AD\u4FE1\u606F .
+errorcode.config_error=DataX\u5F15\u64CE\u914D\u7F6E\u9519\u8BEF\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.secret_error=DataX\u5F15\u64CE\u52A0\u89E3\u5BC6\u51FA\u9519\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.hook_load_error=\u52A0\u8F7D\u5916\u90E8Hook\u51FA\u73B0\u9519\u8BEF\uFF0C\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u5F15\u8D77\u7684
+errorcode.hook_fail_error=\u6267\u884C\u5916\u90E8Hook\u51FA\u73B0\u9519\u8BEF
+errorcode.plugin_install_error=DataX\u63D2\u4EF6\u5B89\u88C5\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_not_found=DataX\u63D2\u4EF6\u914D\u7F6E\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_init_error=DataX\u63D2\u4EF6\u521D\u59CB\u5316\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_runtime_error=DataX\u63D2\u4EF6\u8FD0\u884C\u65F6\u51FA\u9519, \u5177\u4F53\u539F\u56E0\u8BF7\u53C2\u770BDataX\u8FD0\u884C\u7ED3\u675F\u65F6\u7684\u9519\u8BEF\u8BCA\u65AD\u4FE1\u606F .
+errorcode.plugin_dirty_data_limit_exceed=DataX\u4F20\u8F93\u810F\u6570\u636E\u8D85\u8FC7\u7528\u6237\u9884\u671F\uFF0C\u8BE5\u9519\u8BEF\u901A\u5E38\u662F\u7531\u4E8E\u6E90\u7AEF\u6570\u636E\u5B58\u5728\u8F83\u591A\u4E1A\u52A1\u810F\u6570\u636E\u5BFC\u81F4\uFF0C\u8BF7\u4ED4\u7EC6\u68C0\u67E5DataX\u6C47\u62A5\u7684\u810F\u6570\u636E\u65E5\u5FD7\u4FE1\u606F, \u6216\u8005\u60A8\u53EF\u4EE5\u9002\u5F53\u8C03\u5927\u810F\u6570\u636E\u9608\u503C .
+errorcode.plugin_split_error=DataX\u63D2\u4EF6\u5207\u5206\u51FA\u9519, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5404\u4E2A\u63D2\u4EF6\u7F16\u7A0B\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3
+errorcode.kill_job_timeout_error=kill \u4EFB\u52A1\u8D85\u65F6\uFF0C\u8BF7\u8054\u7CFBPE\u89E3\u51B3
+errorcode.start_taskgroup_error=taskGroup\u542F\u52A8\u5931\u8D25,\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3
+errorcode.call_datax_service_failed=\u8BF7\u6C42 DataX Service \u51FA\u9519.
+errorcode.call_remote_failed=\u8FDC\u7A0B\u8C03\u7528\u5931\u8D25
+errorcode.killed_exit_value=Job \u6536\u5230\u4E86 Kill \u547D\u4EE4.
+
+
+httpclientutil.1=\u8BF7\u6C42\u5730\u5740\uFF1A{0}, \u8BF7\u6C42\u65B9\u6CD5\uFF1A{1}, STATUS CODE = {2}, Response Entity: {3}
+httpclientutil.2=\u8FDC\u7A0B\u63A5\u53E3\u8FD4\u56DE-1,\u5C06\u91CD\u8BD5
+
+
+secretutil.1=\u7CFB\u7EDF\u7F16\u7A0B\u9519\u8BEF,\u4E0D\u652F\u6301\u7684\u52A0\u5BC6\u7C7B\u578B
+secretutil.2=\u7CFB\u7EDF\u7F16\u7A0B\u9519\u8BEF,\u4E0D\u652F\u6301\u7684\u52A0\u5BC6\u7C7B\u578B
+secretutil.3=rsa\u52A0\u5BC6\u51FA\u9519
+secretutil.4=rsa\u89E3\u5BC6\u51FA\u9519
+secretutil.5=3\u91CDDES\u52A0\u5BC6\u51FA\u9519
+secretutil.6=rsa\u89E3\u5BC6\u51FA\u9519
+secretutil.7=\u6784\u5EFA\u4E09\u91CDDES\u5BC6\u5319\u51FA\u9519
+secretutil.8=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u65E0\u6CD5\u627E\u5230\u5BC6\u94A5\u7684\u914D\u7F6E\u6587\u4EF6
+secretutil.9=\u8BFB\u53D6\u52A0\u89E3\u5BC6\u914D\u7F6E\u6587\u4EF6\u51FA\u9519
+secretutil.10=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C
+secretutil.11=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7EDF\u7EF4\u62A4\u95EE\u9898
+secretutil.12=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C
+secretutil.13=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7EDF\u7EF4\u62A4\u95EE\u9898
+secretutil.14=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C[{0}]\u5B58\u5728\u5BC6\u94A5\u4E3A\u7A7A\u7684\u60C5\u51B5
+secretutil.15=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u516C\u79C1\u94A5\u5BF9\u5B58\u5728\u4E3A\u7A7A\u7684\u60C5\u51B5\uFF0C\u7248\u672C[{0}]
+secretutil.16=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u65E0\u6CD5\u627E\u5230\u52A0\u89E3\u5BC6\u914D\u7F6E
+
diff --git a/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_en_US.properties b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_en_US.properties
new file mode 100644
index 00000000..8e01b153
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_en_US.properties
@@ -0,0 +1,58 @@
+configparser.1=Failed to load the plug-in [{0},{1}]. We will retry in 1s... Exception: {2}
+configparser.2=Failed to obtain the job configuration information: {0}
+configparser.3=Failed to obtain the job configuration information: {0}
+configparser.4=Failed to obtain the job configuration information: {0}
+configparser.5=Failed to load the plug-in. Loading of the specific plug-in:{0} is not completed
+configparser.6=Failed to load the plug-in. A duplicate plug-in: {0} exists
+
+dataxserviceutil.1=Exception in creating signature. NoSuchAlgorithmException, [{0}]
+dataxserviceutil.2=Exception in creating signature. InvalidKeyException, [{0}]
+dataxserviceutil.3=Exception in creating signature. UnsupportedEncodingException, [{0}]
+
+errorrecordchecker.1=The percentage of dirty data should be limited to within [0.0, 1.0]
+errorrecordchecker.2=The number of dirty data entries should now be a nonnegative integer
+errorrecordchecker.3=Check for the number of dirty data entries has not passed. The limit is [{0}] entries, but [{1}] entries have been captured.
+errorrecordchecker.4=Check for the percentage of dirty data has not passed. The limit is [{0}], but [{1}] of dirty data has been captured.
+
+
+errorcode.install_error=Error in installing DataX engine. Please contact your O&M team to solve the problem.
+errorcode.argument_error=Error in running DataX engine. This problem is generally caused by an internal programming error. Please contact the DataX developer team to solve the problem.
+errorcode.runtime_error=The DataX engine encountered an error during running. For the specific cause, refer to the error diagnosis after DataX stops running.
+errorcode.config_error=Error in DataX engine configuration. This problem is generally caused by a DataX installation error. Please contact your O&M team to solve the problem.
+errorcode.secret_error=Error in DataX engine encryption or decryption. This problem is generally caused by a DataX key configuration error. Please contact your O&M team to solve the problem.
+errorcode.hook_load_error=Error in loading the external hook. This problem is generally caused by the DataX installation.
+errorcode.hook_fail_error=Error in executing the external hook
+errorcode.plugin_install_error=Error in installing DataX plug-in. This problem is generally caused by a DataX installation error. Please contact your O&M team to solve the problem.
+errorcode.plugin_not_found=Error in DataX plug-in configuration. This problem is generally caused by a DataX installation error. Please contact your O&M team to solve the problem.
+errorcode.plugin_init_error=Error in DataX plug-in initialization. This problem is generally caused by a DataX installation error. Please contact your O&M team to solve the problem.
+errorcode.plugin_runtime_error=The DataX plug-in encountered an error during running. For the specific cause, refer to the error diagnosis after DataX stops running.
+errorcode.plugin_dirty_data_limit_exceed=The dirty data transmitted by DataX exceeds user expectations. This error often occurs when a lot dirty data exists in the source data. Please carefully check the dirty data log information reported by DataX, or you can tune up the dirty data threshold value.
+errorcode.plugin_split_error=Error in DataX plug-in slicing. This problem is generally caused by a programming error in some DataX plug-in. Please contact the DataX developer team to solve the problem.
+errorcode.kill_job_timeout_error=The kill task times out. Please contact the PE to solve the problem
+errorcode.start_taskgroup_error=Failed to start the task group. Please contact the DataX developer team to solve the problem
+errorcode.call_datax_service_failed=Error in requesting DataX Service.
+errorcode.call_remote_failed=Remote call failure
+errorcode.killed_exit_value=The job has received a Kill command.
+
+
+httpclientutil.1=Request address: {0}. Request method: {1}. STATUS CODE = {2}, Response Entity: {3}
+httpclientutil.2=The remote interface returns -1. We will try again
+
+
+secretutil.1=System programing error. Unsupported encryption type
+secretutil.2=System programing error. Unsupported encryption type
+secretutil.3=RSA encryption error
+secretutil.4=RSA decryption error
+secretutil.5=Triple DES encryption error
+secretutil.6=RSA decryption error
+secretutil.7=Error in building Triple DES key
+secretutil.8=DataX configuration requires encryption and decryption, but unable to find the key configuration file
+secretutil.9=Error in reading the encryption and decryption configuration file
+secretutil.10=The version of the DataX-configured key is [{0}], but there is no configuration in the system. Error in task key configuration. The key version you configured does not exist
+secretutil.11=The version of the DataX-configured key is [{0}], but there is no configuration in the system. There may be an error in task key configuration, or a problem in system maintenance
+secretutil.12=The version of the DataX-configured key is [{0}], but there is no configuration in the system. Error in task key configuration. The key version you configured does not exist
+secretutil.13=The version of the DataX-configured key is [{0}], but there is no configuration in the system. There may be an error in task key configuration, or a problem in system maintenance
+secretutil.14=DataX configuration requires encryption and decryption, but some key in the configured key version [{0}] is empty
+secretutil.15=DataX configuration requires encryption and decryption, but some configured public/private key pairs are empty and the version is [{0}]
+secretutil.16=DataX configuration requires encryption and decryption, but the encryption and decryption configuration cannot be found
+
diff --git a/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_ja_JP.properties b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_ja_JP.properties
new file mode 100644
index 00000000..7a0c95ac
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_ja_JP.properties
@@ -0,0 +1,58 @@
+configparser.1=\u63D2\u4EF6[{0},{1}]\u52A0\u8F7D\u5931\u8D25\uFF0C1s\u540E\u91CD\u8BD5... Exception:{2}
+configparser.2=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.3=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.4=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.5=\u63D2\u4EF6\u52A0\u8F7D\u5931\u8D25\uFF0C\u672A\u5B8C\u6210\u6307\u5B9A\u63D2\u4EF6\u52A0\u8F7D:{0}
+configparser.6=\u63D2\u4EF6\u52A0\u8F7D\u5931\u8D25,\u5B58\u5728\u91CD\u590D\u63D2\u4EF6:{0}
+
+dataxserviceutil.1=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38NoSuchAlgorithmException, [{0}]
+dataxserviceutil.2=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38InvalidKeyException, [{0}]
+dataxserviceutil.3=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38UnsupportedEncodingException, [{0}]
+
+errorrecordchecker.1=\u810F\u6570\u636E\u767E\u5206\u6BD4\u9650\u5236\u5E94\u8BE5\u5728[0.0, 1.0]\u4E4B\u95F4
+errorrecordchecker.2=\u810F\u6570\u636E\u6761\u6570\u73B0\u5728\u5E94\u8BE5\u4E3A\u975E\u8D1F\u6574\u6570
+errorrecordchecker.3=\u810F\u6570\u636E\u6761\u6570\u68C0\u67E5\u4E0D\u901A\u8FC7\uFF0C\u9650\u5236\u662F[{0}]\u6761\uFF0C\u4F46\u5B9E\u9645\u4E0A\u6355\u83B7\u4E86[{1}]\u6761.
+errorrecordchecker.4=\u810F\u6570\u636E\u767E\u5206\u6BD4\u68C0\u67E5\u4E0D\u901A\u8FC7\uFF0C\u9650\u5236\u662F[{0}]\uFF0C\u4F46\u5B9E\u9645\u4E0A\u6355\u83B7\u5230[{1}].
+
+
+errorcode.install_error=DataX\u5F15\u64CE\u5B89\u88C5\u9519\u8BEF, \u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.argument_error=DataX\u5F15\u64CE\u8FD0\u884C\u9519\u8BEF\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8E\u5185\u90E8\u7F16\u7A0B\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3 .
+errorcode.runtime_error=DataX\u5F15\u64CE\u8FD0\u884C\u8FC7\u7A0B\u51FA\u9519\uFF0C\u5177\u4F53\u539F\u56E0\u8BF7\u53C2\u770BDataX\u8FD0\u884C\u7ED3\u675F\u65F6\u7684\u9519\u8BEF\u8BCA\u65AD\u4FE1\u606F .
+errorcode.config_error=DataX\u5F15\u64CE\u914D\u7F6E\u9519\u8BEF\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.secret_error=DataX\u5F15\u64CE\u52A0\u89E3\u5BC6\u51FA\u9519\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.hook_load_error=\u52A0\u8F7D\u5916\u90E8Hook\u51FA\u73B0\u9519\u8BEF\uFF0C\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u5F15\u8D77\u7684
+errorcode.hook_fail_error=\u6267\u884C\u5916\u90E8Hook\u51FA\u73B0\u9519\u8BEF
+errorcode.plugin_install_error=DataX\u63D2\u4EF6\u5B89\u88C5\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_not_found=DataX\u63D2\u4EF6\u914D\u7F6E\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_init_error=DataX\u63D2\u4EF6\u521D\u59CB\u5316\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_runtime_error=DataX\u63D2\u4EF6\u8FD0\u884C\u65F6\u51FA\u9519, \u5177\u4F53\u539F\u56E0\u8BF7\u53C2\u770BDataX\u8FD0\u884C\u7ED3\u675F\u65F6\u7684\u9519\u8BEF\u8BCA\u65AD\u4FE1\u606F .
+errorcode.plugin_dirty_data_limit_exceed=DataX\u4F20\u8F93\u810F\u6570\u636E\u8D85\u8FC7\u7528\u6237\u9884\u671F\uFF0C\u8BE5\u9519\u8BEF\u901A\u5E38\u662F\u7531\u4E8E\u6E90\u7AEF\u6570\u636E\u5B58\u5728\u8F83\u591A\u4E1A\u52A1\u810F\u6570\u636E\u5BFC\u81F4\uFF0C\u8BF7\u4ED4\u7EC6\u68C0\u67E5DataX\u6C47\u62A5\u7684\u810F\u6570\u636E\u65E5\u5FD7\u4FE1\u606F, \u6216\u8005\u60A8\u53EF\u4EE5\u9002\u5F53\u8C03\u5927\u810F\u6570\u636E\u9608\u503C .
+errorcode.plugin_split_error=DataX\u63D2\u4EF6\u5207\u5206\u51FA\u9519, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5404\u4E2A\u63D2\u4EF6\u7F16\u7A0B\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3
+errorcode.kill_job_timeout_error=kill \u4EFB\u52A1\u8D85\u65F6\uFF0C\u8BF7\u8054\u7CFBPE\u89E3\u51B3
+errorcode.start_taskgroup_error=taskGroup\u542F\u52A8\u5931\u8D25,\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3
+errorcode.call_datax_service_failed=\u8BF7\u6C42 DataX Service \u51FA\u9519.
+errorcode.call_remote_failed=\u8FDC\u7A0B\u8C03\u7528\u5931\u8D25
+errorcode.killed_exit_value=Job \u6536\u5230\u4E86 Kill \u547D\u4EE4.
+
+
+httpclientutil.1=\u8BF7\u6C42\u5730\u5740\uFF1A{0}, \u8BF7\u6C42\u65B9\u6CD5\uFF1A{1},STATUS CODE = {2}, Response Entity: {3}
+httpclientutil.2=\u8FDC\u7A0B\u63A5\u53E3\u8FD4\u56DE-1,\u5C06\u91CD\u8BD5
+
+
+secretutil.1=\u7CFB\u7EDF\u7F16\u7A0B\u9519\u8BEF,\u4E0D\u652F\u6301\u7684\u52A0\u5BC6\u7C7B\u578B
+secretutil.2=\u7CFB\u7EDF\u7F16\u7A0B\u9519\u8BEF,\u4E0D\u652F\u6301\u7684\u52A0\u5BC6\u7C7B\u578B
+secretutil.3=rsa\u52A0\u5BC6\u51FA\u9519
+secretutil.4=rsa\u89E3\u5BC6\u51FA\u9519
+secretutil.5=3\u91CDDES\u52A0\u5BC6\u51FA\u9519
+secretutil.6=rsa\u89E3\u5BC6\u51FA\u9519
+secretutil.7=\u6784\u5EFA\u4E09\u91CDDES\u5BC6\u5319\u51FA\u9519
+secretutil.8=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u65E0\u6CD5\u627E\u5230\u5BC6\u94A5\u7684\u914D\u7F6E\u6587\u4EF6
+secretutil.9=\u8BFB\u53D6\u52A0\u89E3\u5BC6\u914D\u7F6E\u6587\u4EF6\u51FA\u9519
+secretutil.10=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C
+secretutil.11=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7EDF\u7EF4\u62A4\u95EE\u9898
+secretutil.12=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C
+secretutil.13=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7EDF\u7EF4\u62A4\u95EE\u9898
+secretutil.14=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C[{0}]\u5B58\u5728\u5BC6\u94A5\u4E3A\u7A7A\u7684\u60C5\u51B5
+secretutil.15=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u516C\u79C1\u94A5\u5BF9\u5B58\u5728\u4E3A\u7A7A\u7684\u60C5\u51B5\uFF0C\u7248\u672C[{0}]
+secretutil.16=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u65E0\u6CD5\u627E\u5230\u52A0\u89E3\u5BC6\u914D\u7F6E
+
diff --git a/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_zh_CN.properties b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_zh_CN.properties
new file mode 100644
index 00000000..7a0c95ac
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_zh_CN.properties
@@ -0,0 +1,58 @@
+configparser.1=\u63D2\u4EF6[{0},{1}]\u52A0\u8F7D\u5931\u8D25\uFF0C1s\u540E\u91CD\u8BD5... Exception:{2}
+configparser.2=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.3=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.4=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.5=\u63D2\u4EF6\u52A0\u8F7D\u5931\u8D25\uFF0C\u672A\u5B8C\u6210\u6307\u5B9A\u63D2\u4EF6\u52A0\u8F7D:{0}
+configparser.6=\u63D2\u4EF6\u52A0\u8F7D\u5931\u8D25,\u5B58\u5728\u91CD\u590D\u63D2\u4EF6:{0}
+
+dataxserviceutil.1=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38NoSuchAlgorithmException, [{0}]
+dataxserviceutil.2=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38InvalidKeyException, [{0}]
+dataxserviceutil.3=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38UnsupportedEncodingException, [{0}]
+
+errorrecordchecker.1=\u810F\u6570\u636E\u767E\u5206\u6BD4\u9650\u5236\u5E94\u8BE5\u5728[0.0, 1.0]\u4E4B\u95F4
+errorrecordchecker.2=\u810F\u6570\u636E\u6761\u6570\u73B0\u5728\u5E94\u8BE5\u4E3A\u975E\u8D1F\u6574\u6570
+errorrecordchecker.3=\u810F\u6570\u636E\u6761\u6570\u68C0\u67E5\u4E0D\u901A\u8FC7\uFF0C\u9650\u5236\u662F[{0}]\u6761\uFF0C\u4F46\u5B9E\u9645\u4E0A\u6355\u83B7\u4E86[{1}]\u6761.
+errorrecordchecker.4=\u810F\u6570\u636E\u767E\u5206\u6BD4\u68C0\u67E5\u4E0D\u901A\u8FC7\uFF0C\u9650\u5236\u662F[{0}]\uFF0C\u4F46\u5B9E\u9645\u4E0A\u6355\u83B7\u5230[{1}].
+
+
+errorcode.install_error=DataX\u5F15\u64CE\u5B89\u88C5\u9519\u8BEF, \u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.argument_error=DataX\u5F15\u64CE\u8FD0\u884C\u9519\u8BEF\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8E\u5185\u90E8\u7F16\u7A0B\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3 .
+errorcode.runtime_error=DataX\u5F15\u64CE\u8FD0\u884C\u8FC7\u7A0B\u51FA\u9519\uFF0C\u5177\u4F53\u539F\u56E0\u8BF7\u53C2\u770BDataX\u8FD0\u884C\u7ED3\u675F\u65F6\u7684\u9519\u8BEF\u8BCA\u65AD\u4FE1\u606F .
+errorcode.config_error=DataX\u5F15\u64CE\u914D\u7F6E\u9519\u8BEF\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.secret_error=DataX\u5F15\u64CE\u52A0\u89E3\u5BC6\u51FA\u9519\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.hook_load_error=\u52A0\u8F7D\u5916\u90E8Hook\u51FA\u73B0\u9519\u8BEF\uFF0C\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u5F15\u8D77\u7684
+errorcode.hook_fail_error=\u6267\u884C\u5916\u90E8Hook\u51FA\u73B0\u9519\u8BEF
+errorcode.plugin_install_error=DataX\u63D2\u4EF6\u5B89\u88C5\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_not_found=DataX\u63D2\u4EF6\u914D\u7F6E\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_init_error=DataX\u63D2\u4EF6\u521D\u59CB\u5316\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_runtime_error=DataX\u63D2\u4EF6\u8FD0\u884C\u65F6\u51FA\u9519, \u5177\u4F53\u539F\u56E0\u8BF7\u53C2\u770BDataX\u8FD0\u884C\u7ED3\u675F\u65F6\u7684\u9519\u8BEF\u8BCA\u65AD\u4FE1\u606F .
+errorcode.plugin_dirty_data_limit_exceed=DataX\u4F20\u8F93\u810F\u6570\u636E\u8D85\u8FC7\u7528\u6237\u9884\u671F\uFF0C\u8BE5\u9519\u8BEF\u901A\u5E38\u662F\u7531\u4E8E\u6E90\u7AEF\u6570\u636E\u5B58\u5728\u8F83\u591A\u4E1A\u52A1\u810F\u6570\u636E\u5BFC\u81F4\uFF0C\u8BF7\u4ED4\u7EC6\u68C0\u67E5DataX\u6C47\u62A5\u7684\u810F\u6570\u636E\u65E5\u5FD7\u4FE1\u606F, \u6216\u8005\u60A8\u53EF\u4EE5\u9002\u5F53\u8C03\u5927\u810F\u6570\u636E\u9608\u503C .
+errorcode.plugin_split_error=DataX\u63D2\u4EF6\u5207\u5206\u51FA\u9519, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5404\u4E2A\u63D2\u4EF6\u7F16\u7A0B\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3
+errorcode.kill_job_timeout_error=kill \u4EFB\u52A1\u8D85\u65F6\uFF0C\u8BF7\u8054\u7CFBPE\u89E3\u51B3
+errorcode.start_taskgroup_error=taskGroup\u542F\u52A8\u5931\u8D25,\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3
+errorcode.call_datax_service_failed=\u8BF7\u6C42 DataX Service \u51FA\u9519.
+errorcode.call_remote_failed=\u8FDC\u7A0B\u8C03\u7528\u5931\u8D25
+errorcode.killed_exit_value=Job \u6536\u5230\u4E86 Kill \u547D\u4EE4.
+
+
+httpclientutil.1=\u8BF7\u6C42\u5730\u5740\uFF1A{0}, \u8BF7\u6C42\u65B9\u6CD5\uFF1A{1},STATUS CODE = {2}, Response Entity: {3}
+httpclientutil.2=\u8FDC\u7A0B\u63A5\u53E3\u8FD4\u56DE-1,\u5C06\u91CD\u8BD5
+
+
+secretutil.1=\u7CFB\u7EDF\u7F16\u7A0B\u9519\u8BEF,\u4E0D\u652F\u6301\u7684\u52A0\u5BC6\u7C7B\u578B
+secretutil.2=\u7CFB\u7EDF\u7F16\u7A0B\u9519\u8BEF,\u4E0D\u652F\u6301\u7684\u52A0\u5BC6\u7C7B\u578B
+secretutil.3=rsa\u52A0\u5BC6\u51FA\u9519
+secretutil.4=rsa\u89E3\u5BC6\u51FA\u9519
+secretutil.5=3\u91CDDES\u52A0\u5BC6\u51FA\u9519
+secretutil.6=rsa\u89E3\u5BC6\u51FA\u9519
+secretutil.7=\u6784\u5EFA\u4E09\u91CDDES\u5BC6\u5319\u51FA\u9519
+secretutil.8=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u65E0\u6CD5\u627E\u5230\u5BC6\u94A5\u7684\u914D\u7F6E\u6587\u4EF6
+secretutil.9=\u8BFB\u53D6\u52A0\u89E3\u5BC6\u914D\u7F6E\u6587\u4EF6\u51FA\u9519
+secretutil.10=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C
+secretutil.11=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7EDF\u7EF4\u62A4\u95EE\u9898
+secretutil.12=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C
+secretutil.13=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7EDF\u7EF4\u62A4\u95EE\u9898
+secretutil.14=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C[{0}]\u5B58\u5728\u5BC6\u94A5\u4E3A\u7A7A\u7684\u60C5\u51B5
+secretutil.15=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u516C\u79C1\u94A5\u5BF9\u5B58\u5728\u4E3A\u7A7A\u7684\u60C5\u51B5\uFF0C\u7248\u672C[{0}]
+secretutil.16=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u65E0\u6CD5\u627E\u5230\u52A0\u89E3\u5BC6\u914D\u7F6E
+
diff --git a/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_zh_HK.properties b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_zh_HK.properties
new file mode 100644
index 00000000..59ce9fd9
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_zh_HK.properties
@@ -0,0 +1,116 @@
+configparser.1=\u63D2\u4EF6[{0},{1}]\u52A0\u8F7D\u5931\u8D25\uFF0C1s\u540E\u91CD\u8BD5... Exception:{2}
+configparser.2=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.3=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.4=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.5=\u63D2\u4EF6\u52A0\u8F7D\u5931\u8D25\uFF0C\u672A\u5B8C\u6210\u6307\u5B9A\u63D2\u4EF6\u52A0\u8F7D:{0}
+configparser.6=\u63D2\u4EF6\u52A0\u8F7D\u5931\u8D25,\u5B58\u5728\u91CD\u590D\u63D2\u4EF6:{0}
+
+dataxserviceutil.1=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38NoSuchAlgorithmException, [{0}]
+dataxserviceutil.2=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38InvalidKeyException, [{0}]
+dataxserviceutil.3=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38UnsupportedEncodingException, [{0}]
+
+errorrecordchecker.1=\u810F\u6570\u636E\u767E\u5206\u6BD4\u9650\u5236\u5E94\u8BE5\u5728[0.0, 1.0]\u4E4B\u95F4
+errorrecordchecker.2=\u810F\u6570\u636E\u6761\u6570\u73B0\u5728\u5E94\u8BE5\u4E3A\u975E\u8D1F\u6574\u6570
+errorrecordchecker.3=\u810F\u6570\u636E\u6761\u6570\u68C0\u67E5\u4E0D\u901A\u8FC7\uFF0C\u9650\u5236\u662F[{0}]\u6761\uFF0C\u4F46\u5B9E\u9645\u4E0A\u6355\u83B7\u4E86[{1}]\u6761.
+errorrecordchecker.4=\u810F\u6570\u636E\u767E\u5206\u6BD4\u68C0\u67E5\u4E0D\u901A\u8FC7\uFF0C\u9650\u5236\u662F[{0}]\uFF0C\u4F46\u5B9E\u9645\u4E0A\u6355\u83B7\u5230[{1}].
+
+
+errorcode.install_error=DataX\u5F15\u64CE\u5B89\u88C5\u9519\u8BEF, \u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.argument_error=DataX\u5F15\u64CE\u8FD0\u884C\u9519\u8BEF\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8E\u5185\u90E8\u7F16\u7A0B\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3 .
+errorcode.runtime_error=DataX\u5F15\u64CE\u8FD0\u884C\u8FC7\u7A0B\u51FA\u9519\uFF0C\u5177\u4F53\u539F\u56E0\u8BF7\u53C2\u770BDataX\u8FD0\u884C\u7ED3\u675F\u65F6\u7684\u9519\u8BEF\u8BCA\u65AD\u4FE1\u606F .
+errorcode.config_error=DataX\u5F15\u64CE\u914D\u7F6E\u9519\u8BEF\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.secret_error=DataX\u5F15\u64CE\u52A0\u89E3\u5BC6\u51FA\u9519\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.hook_load_error=\u52A0\u8F7D\u5916\u90E8Hook\u51FA\u73B0\u9519\u8BEF\uFF0C\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u5F15\u8D77\u7684
+errorcode.hook_fail_error=\u6267\u884C\u5916\u90E8Hook\u51FA\u73B0\u9519\u8BEF
+errorcode.plugin_install_error=DataX\u63D2\u4EF6\u5B89\u88C5\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_not_found=DataX\u63D2\u4EF6\u914D\u7F6E\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_init_error=DataX\u63D2\u4EF6\u521D\u59CB\u5316\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_runtime_error=DataX\u63D2\u4EF6\u8FD0\u884C\u65F6\u51FA\u9519, \u5177\u4F53\u539F\u56E0\u8BF7\u53C2\u770BDataX\u8FD0\u884C\u7ED3\u675F\u65F6\u7684\u9519\u8BEF\u8BCA\u65AD\u4FE1\u606F .
+errorcode.plugin_dirty_data_limit_exceed=DataX\u4F20\u8F93\u810F\u6570\u636E\u8D85\u8FC7\u7528\u6237\u9884\u671F\uFF0C\u8BE5\u9519\u8BEF\u901A\u5E38\u662F\u7531\u4E8E\u6E90\u7AEF\u6570\u636E\u5B58\u5728\u8F83\u591A\u4E1A\u52A1\u810F\u6570\u636E\u5BFC\u81F4\uFF0C\u8BF7\u4ED4\u7EC6\u68C0\u67E5DataX\u6C47\u62A5\u7684\u810F\u6570\u636E\u65E5\u5FD7\u4FE1\u606F, \u6216\u8005\u60A8\u53EF\u4EE5\u9002\u5F53\u8C03\u5927\u810F\u6570\u636E\u9608\u503C .
+errorcode.plugin_split_error=DataX\u63D2\u4EF6\u5207\u5206\u51FA\u9519, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5404\u4E2A\u63D2\u4EF6\u7F16\u7A0B\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3
+errorcode.kill_job_timeout_error=kill \u4EFB\u52A1\u8D85\u65F6\uFF0C\u8BF7\u8054\u7CFBPE\u89E3\u51B3
+errorcode.start_taskgroup_error=taskGroup\u542F\u52A8\u5931\u8D25,\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3
+errorcode.call_datax_service_failed=\u8BF7\u6C42 DataX Service \u51FA\u9519.
+errorcode.call_remote_failed=\u8FDC\u7A0B\u8C03\u7528\u5931\u8D25
+errorcode.killed_exit_value=Job \u6536\u5230\u4E86 Kill \u547D\u4EE4.
+
+
+httpclientutil.1=\u8BF7\u6C42\u5730\u5740\uFF1A{0}, \u8BF7\u6C42\u65B9\u6CD5\uFF1A{1},STATUS CODE = {2}, Response Entity: {3}
+httpclientutil.2=\u8FDC\u7A0B\u63A5\u53E3\u8FD4\u56DE-1,\u5C06\u91CD\u8BD5
+
+
+secretutil.1=\u7CFB\u7EDF\u7F16\u7A0B\u9519\u8BEF,\u4E0D\u652F\u6301\u7684\u52A0\u5BC6\u7C7B\u578B
+secretutil.2=\u7CFB\u7EDF\u7F16\u7A0B\u9519\u8BEF,\u4E0D\u652F\u6301\u7684\u52A0\u5BC6\u7C7B\u578B
+secretutil.3=rsa\u52A0\u5BC6\u51FA\u9519
+secretutil.4=rsa\u89E3\u5BC6\u51FA\u9519
+secretutil.5=3\u91CDDES\u52A0\u5BC6\u51FA\u9519
+secretutil.6=rsa\u89E3\u5BC6\u51FA\u9519
+secretutil.7=\u6784\u5EFA\u4E09\u91CDDES\u5BC6\u5319\u51FA\u9519
+secretutil.8=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u65E0\u6CD5\u627E\u5230\u5BC6\u94A5\u7684\u914D\u7F6E\u6587\u4EF6
+secretutil.9=\u8BFB\u53D6\u52A0\u89E3\u5BC6\u914D\u7F6E\u6587\u4EF6\u51FA\u9519
+secretutil.10=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C
+secretutil.11=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7EDF\u7EF4\u62A4\u95EE\u9898
+secretutil.12=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C
+secretutil.13=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7EDF\u7EF4\u62A4\u95EE\u9898
+secretutil.14=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C[{0}]\u5B58\u5728\u5BC6\u94A5\u4E3A\u7A7A\u7684\u60C5\u51B5
+secretutil.15=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u516C\u79C1\u94A5\u5BF9\u5B58\u5728\u4E3A\u7A7A\u7684\u60C5\u51B5\uFF0C\u7248\u672C[{0}]
+secretutil.16=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u65E0\u6CD5\u627E\u5230\u52A0\u89E3\u5BC6\u914D\u7F6E
+
+configparser.1=\u5916\u639B\u7A0B\u5F0F[{0},{1}]\u8F09\u5165\u5931\u6557\uFF0C1s\u5F8C\u91CD\u8A66... Exception:{2}
+configparser.2=\u7372\u53D6\u4F5C\u696D\u914D\u7F6E\u8CC7\u8A0A\u5931\u6557:{0}
+configparser.3=\u7372\u53D6\u4F5C\u696D\u914D\u7F6E\u8CC7\u8A0A\u5931\u6557:{0}
+configparser.4=\u7372\u53D6\u4F5C\u696D\u914D\u7F6E\u8CC7\u8A0A\u5931\u6557:{0}
+configparser.5=\u5916\u639B\u7A0B\u5F0F\u8F09\u5165\u5931\u6557\uFF0C\u672A\u5B8C\u6210\u6307\u5B9A\u5916\u639B\u7A0B\u5F0F\u8F09\u5165:{0}
+configparser.6=\u5916\u639B\u7A0B\u5F0F\u8F09\u5165\u5931\u6557,\u5B58\u5728\u91CD\u8907\u5916\u639B\u7A0B\u5F0F:{0}
+
+dataxserviceutil.1=\u5EFA\u7ACB\u7C3D\u540D\u7570\u5E38NoSuchAlgorithmException, [{0}]
+dataxserviceutil.2=\u5EFA\u7ACB\u7C3D\u540D\u7570\u5E38InvalidKeyException, [{0}]
+dataxserviceutil.3=\u5EFA\u7ACB\u7C3D\u540D\u7570\u5E38UnsupportedEncodingException, [{0}]
+
+errorrecordchecker.1=\u9AD2\u6578\u64DA\u767E\u5206\u6BD4\u9650\u5236\u61C9\u8A72\u5728[0.0, 1.0]\u4E4B\u9593
+errorrecordchecker.2=\u9AD2\u6578\u64DA\u689D\u6578\u73FE\u5728\u61C9\u8A72\u70BA\u975E\u8CA0\u6574\u6578
+errorrecordchecker.3=\u9AD2\u6578\u64DA\u689D\u6578\u6AA2\u67E5\u4E0D\u901A\u904E\uFF0C\u9650\u5236\u662F[{0}]\u689D\uFF0C\u4F46\u5BE6\u969B\u4E0A\u6355\u7372\u4E86[{1}]\u689D.
+errorrecordchecker.4=\u9AD2\u6578\u64DA\u767E\u5206\u6BD4\u6AA2\u67E5\u4E0D\u901A\u904E\uFF0C\u9650\u5236\u662F[{0}]\uFF0C\u4F46\u5BE6\u969B\u4E0A\u6355\u7372\u5230[{1}].
+
+
+errorcode.install_error=DataX\u5F15\u64CE\u5B89\u88DD\u932F\u8AA4, \u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.argument_error=DataX\u5F15\u64CE\u904B\u884C\u932F\u8AA4\uFF0C\u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BC\u5167\u90E8\u7DE8\u7A0B\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61DataX\u958B\u767C\u5718\u968A\u89E3\u6C7A .
+errorcode.runtime_error=DataX\u5F15\u64CE\u904B\u884C\u904E\u7A0B\u51FA\u932F\uFF0C\u5177\u9AD4\u539F\u56E0\u8ACB\u53C3\u770BDataX\u904B\u884C\u7D50\u675F\u6642\u7684\u932F\u8AA4\u8A3A\u65B7\u8CC7\u8A0A .
+errorcode.config_error=DataX\u5F15\u64CE\u914D\u7F6E\u932F\u8AA4\uFF0C\u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5B89\u88DD\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.secret_error=DataX\u5F15\u64CE\u52A0\u89E3\u5BC6\u51FA\u932F\uFF0C\u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5BC6\u9470\u914D\u7F6E\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.hook_load_error=\u8F09\u5165\u5916\u90E8Hook\u51FA\u73FE\u932F\u8AA4\uFF0C\u901A\u5E38\u662F\u7531\u65BCDataX\u5B89\u88DD\u5F15\u8D77\u7684
+errorcode.hook_fail_error=\u57F7\u884C\u5916\u90E8Hook\u51FA\u73FE\u932F\u8AA4
+errorcode.plugin_install_error=DataX\u5916\u639B\u7A0B\u5F0F\u5B89\u88DD\u932F\u8AA4, \u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5B89\u88DD\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.plugin_not_found=DataX\u5916\u639B\u7A0B\u5F0F\u914D\u7F6E\u932F\u8AA4, \u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5B89\u88DD\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.plugin_init_error=DataX\u5916\u639B\u7A0B\u5F0F\u521D\u59CB\u5316\u932F\u8AA4, \u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5B89\u88DD\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.plugin_runtime_error=DataX\u5916\u639B\u7A0B\u5F0F\u904B\u884C\u6642\u51FA\u932F, \u5177\u9AD4\u539F\u56E0\u8ACB\u53C3\u770BDataX\u904B\u884C\u7D50\u675F\u6642\u7684\u932F\u8AA4\u8A3A\u65B7\u8CC7\u8A0A .
+errorcode.plugin_dirty_data_limit_exceed=DataX\u50B3\u8F38\u9AD2\u6578\u64DA\u8D85\u904E\u7528\u6236\u9810\u671F\uFF0C\u8A72\u932F\u8AA4\u901A\u5E38\u662F\u7531\u65BC\u6E90\u7AEF\u6578\u64DA\u5B58\u5728\u8F03\u591A\u696D\u52D9\u9AD2\u6578\u64DA\u5C0E\u81F4\uFF0C\u8ACB\u4ED4\u7D30\u6AA2\u67E5DataX\u5F59\u5831\u7684\u9AD2\u6578\u64DA\u65E5\u8A8C\u8CC7\u8A0A, \u6216\u8005\u60A8\u53EF\u4EE5\u9069\u7576\u8ABF\u5927\u9AD2\u6578\u64DA\u95BE\u503C .
+errorcode.plugin_split_error=DataX\u5916\u639B\u7A0B\u5F0F\u5207\u5206\u51FA\u932F, \u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5404\u500B\u5916\u639B\u7A0B\u5F0F\u7DE8\u7A0B\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61DataX\u958B\u767C\u5718\u968A\u89E3\u6C7A
+errorcode.kill_job_timeout_error=kill \u4EFB\u52D9\u903E\u6642\uFF0C\u8ACB\u806F\u7D61PE\u89E3\u6C7A
+errorcode.start_taskgroup_error=taskGroup\u555F\u52D5\u5931\u6557,\u8ACB\u806F\u7D61DataX\u958B\u767C\u5718\u968A\u89E3\u6C7A
+errorcode.call_datax_service_failed=\u8ACB\u6C42 DataX Service \u51FA\u932F.
+errorcode.call_remote_failed=\u9060\u7A0B\u8ABF\u7528\u5931\u6557
+errorcode.killed_exit_value=Job \u6536\u5230\u4E86 Kill \u547D\u4EE4.
+
+
+httpclientutil.1=\u8ACB\u6C42\u5730\u5740\uFF1A{0}, \u8ACB\u6C42\u65B9\u6CD5\uFF1A{1},STATUS CODE = {2}, Response Entity: {3}
+httpclientutil.2=\u9060\u7A0B\u63A5\u53E3\u8FD4\u56DE-1,\u5C07\u91CD\u8A66
+
+
+secretutil.1=\u7CFB\u7D71\u7DE8\u7A0B\u932F\u8AA4,\u4E0D\u652F\u63F4\u7684\u52A0\u5BC6\u985E\u578B
+secretutil.2=\u7CFB\u7D71\u7DE8\u7A0B\u932F\u8AA4,\u4E0D\u652F\u63F4\u7684\u52A0\u5BC6\u985E\u578B
+secretutil.3=rsa\u52A0\u5BC6\u51FA\u932F
+secretutil.4=rsa\u89E3\u5BC6\u51FA\u932F
+secretutil.5=3\u91CDDES\u52A0\u5BC6\u51FA\u932F
+secretutil.6=rsa\u89E3\u5BC6\u51FA\u932F
+secretutil.7=\u69CB\u5EFA\u4E09\u91CDDES\u5BC6\u5319\u51FA\u932F
+secretutil.8=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u7121\u6CD5\u627E\u5230\u5BC6\u9470\u7684\u914D\u7F6E\u6A94\u6848
+secretutil.9=\u8B80\u53D6\u52A0\u89E3\u5BC6\u914D\u7F6E\u6A94\u6848\u51FA\u932F
+secretutil.10=DataX\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C\u70BA[{0}]\uFF0C\u4F46\u5728\u7CFB\u7D71\u4E2D\u6C92\u6709\u914D\u7F6E\uFF0C\u4EFB\u52D9\u5BC6\u9470\u914D\u7F6E\u932F\u8AA4\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C
+secretutil.11=DataX\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C\u70BA[{0}]\uFF0C\u4F46\u5728\u7CFB\u7D71\u4E2D\u6C92\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52D9\u5BC6\u9470\u914D\u7F6E\u932F\u8AA4\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7D71\u7DAD\u8B77\u554F\u984C
+secretutil.12=DataX\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C\u70BA[{0}]\uFF0C\u4F46\u5728\u7CFB\u7D71\u4E2D\u6C92\u6709\u914D\u7F6E\uFF0C\u4EFB\u52D9\u5BC6\u9470\u914D\u7F6E\u932F\u8AA4\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C
+secretutil.13=DataX\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C\u70BA[{0}]\uFF0C\u4F46\u5728\u7CFB\u7D71\u4E2D\u6C92\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52D9\u5BC6\u9470\u914D\u7F6E\u932F\u8AA4\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7D71\u7DAD\u8B77\u554F\u984C
+secretutil.14=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C[{0}]\u5B58\u5728\u5BC6\u9470\u70BA\u7A7A\u7684\u60C5\u6CC1
+secretutil.15=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u516C\u79C1\u9470\u5C0D\u5B58\u5728\u70BA\u7A7A\u7684\u60C5\u6CC1\uFF0C\u7248\u672C[{0}]
+secretutil.16=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u7121\u6CD5\u627E\u5230\u52A0\u89E3\u5BC6\u914D\u7F6E
+
diff --git a/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_zh_TW.properties b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_zh_TW.properties
new file mode 100644
index 00000000..59ce9fd9
--- /dev/null
+++ b/core/src/main/java/com/alibaba/datax/core/util/LocalStrings_zh_TW.properties
@@ -0,0 +1,116 @@
+configparser.1=\u63D2\u4EF6[{0},{1}]\u52A0\u8F7D\u5931\u8D25\uFF0C1s\u540E\u91CD\u8BD5... Exception:{2}
+configparser.2=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.3=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.4=\u83B7\u53D6\u4F5C\u4E1A\u914D\u7F6E\u4FE1\u606F\u5931\u8D25:{0}
+configparser.5=\u63D2\u4EF6\u52A0\u8F7D\u5931\u8D25\uFF0C\u672A\u5B8C\u6210\u6307\u5B9A\u63D2\u4EF6\u52A0\u8F7D:{0}
+configparser.6=\u63D2\u4EF6\u52A0\u8F7D\u5931\u8D25,\u5B58\u5728\u91CD\u590D\u63D2\u4EF6:{0}
+
+dataxserviceutil.1=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38NoSuchAlgorithmException, [{0}]
+dataxserviceutil.2=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38InvalidKeyException, [{0}]
+dataxserviceutil.3=\u521B\u5EFA\u7B7E\u540D\u5F02\u5E38UnsupportedEncodingException, [{0}]
+
+errorrecordchecker.1=\u810F\u6570\u636E\u767E\u5206\u6BD4\u9650\u5236\u5E94\u8BE5\u5728[0.0, 1.0]\u4E4B\u95F4
+errorrecordchecker.2=\u810F\u6570\u636E\u6761\u6570\u73B0\u5728\u5E94\u8BE5\u4E3A\u975E\u8D1F\u6574\u6570
+errorrecordchecker.3=\u810F\u6570\u636E\u6761\u6570\u68C0\u67E5\u4E0D\u901A\u8FC7\uFF0C\u9650\u5236\u662F[{0}]\u6761\uFF0C\u4F46\u5B9E\u9645\u4E0A\u6355\u83B7\u4E86[{1}]\u6761.
+errorrecordchecker.4=\u810F\u6570\u636E\u767E\u5206\u6BD4\u68C0\u67E5\u4E0D\u901A\u8FC7\uFF0C\u9650\u5236\u662F[{0}]\uFF0C\u4F46\u5B9E\u9645\u4E0A\u6355\u83B7\u5230[{1}].
+
+
+errorcode.install_error=DataX\u5F15\u64CE\u5B89\u88C5\u9519\u8BEF, \u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.argument_error=DataX\u5F15\u64CE\u8FD0\u884C\u9519\u8BEF\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8E\u5185\u90E8\u7F16\u7A0B\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3 .
+errorcode.runtime_error=DataX\u5F15\u64CE\u8FD0\u884C\u8FC7\u7A0B\u51FA\u9519\uFF0C\u5177\u4F53\u539F\u56E0\u8BF7\u53C2\u770BDataX\u8FD0\u884C\u7ED3\u675F\u65F6\u7684\u9519\u8BEF\u8BCA\u65AD\u4FE1\u606F .
+errorcode.config_error=DataX\u5F15\u64CE\u914D\u7F6E\u9519\u8BEF\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.secret_error=DataX\u5F15\u64CE\u52A0\u89E3\u5BC6\u51FA\u9519\uFF0C\u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.hook_load_error=\u52A0\u8F7D\u5916\u90E8Hook\u51FA\u73B0\u9519\u8BEF\uFF0C\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u5F15\u8D77\u7684
+errorcode.hook_fail_error=\u6267\u884C\u5916\u90E8Hook\u51FA\u73B0\u9519\u8BEF
+errorcode.plugin_install_error=DataX\u63D2\u4EF6\u5B89\u88C5\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_not_found=DataX\u63D2\u4EF6\u914D\u7F6E\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_init_error=DataX\u63D2\u4EF6\u521D\u59CB\u5316\u9519\u8BEF, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5B89\u88C5\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFB\u60A8\u7684\u8FD0\u7EF4\u89E3\u51B3 .
+errorcode.plugin_runtime_error=DataX\u63D2\u4EF6\u8FD0\u884C\u65F6\u51FA\u9519, \u5177\u4F53\u539F\u56E0\u8BF7\u53C2\u770BDataX\u8FD0\u884C\u7ED3\u675F\u65F6\u7684\u9519\u8BEF\u8BCA\u65AD\u4FE1\u606F .
+errorcode.plugin_dirty_data_limit_exceed=DataX\u4F20\u8F93\u810F\u6570\u636E\u8D85\u8FC7\u7528\u6237\u9884\u671F\uFF0C\u8BE5\u9519\u8BEF\u901A\u5E38\u662F\u7531\u4E8E\u6E90\u7AEF\u6570\u636E\u5B58\u5728\u8F83\u591A\u4E1A\u52A1\u810F\u6570\u636E\u5BFC\u81F4\uFF0C\u8BF7\u4ED4\u7EC6\u68C0\u67E5DataX\u6C47\u62A5\u7684\u810F\u6570\u636E\u65E5\u5FD7\u4FE1\u606F, \u6216\u8005\u60A8\u53EF\u4EE5\u9002\u5F53\u8C03\u5927\u810F\u6570\u636E\u9608\u503C .
+errorcode.plugin_split_error=DataX\u63D2\u4EF6\u5207\u5206\u51FA\u9519, \u8BE5\u95EE\u9898\u901A\u5E38\u662F\u7531\u4E8EDataX\u5404\u4E2A\u63D2\u4EF6\u7F16\u7A0B\u9519\u8BEF\u5F15\u8D77\uFF0C\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3
+errorcode.kill_job_timeout_error=kill \u4EFB\u52A1\u8D85\u65F6\uFF0C\u8BF7\u8054\u7CFBPE\u89E3\u51B3
+errorcode.start_taskgroup_error=taskGroup\u542F\u52A8\u5931\u8D25,\u8BF7\u8054\u7CFBDataX\u5F00\u53D1\u56E2\u961F\u89E3\u51B3
+errorcode.call_datax_service_failed=\u8BF7\u6C42 DataX Service \u51FA\u9519.
+errorcode.call_remote_failed=\u8FDC\u7A0B\u8C03\u7528\u5931\u8D25
+errorcode.killed_exit_value=Job \u6536\u5230\u4E86 Kill \u547D\u4EE4.
+
+
+httpclientutil.1=\u8BF7\u6C42\u5730\u5740\uFF1A{0}, \u8BF7\u6C42\u65B9\u6CD5\uFF1A{1},STATUS CODE = {2}, Response Entity: {3}
+httpclientutil.2=\u8FDC\u7A0B\u63A5\u53E3\u8FD4\u56DE-1,\u5C06\u91CD\u8BD5
+
+
+secretutil.1=\u7CFB\u7EDF\u7F16\u7A0B\u9519\u8BEF,\u4E0D\u652F\u6301\u7684\u52A0\u5BC6\u7C7B\u578B
+secretutil.2=\u7CFB\u7EDF\u7F16\u7A0B\u9519\u8BEF,\u4E0D\u652F\u6301\u7684\u52A0\u5BC6\u7C7B\u578B
+secretutil.3=rsa\u52A0\u5BC6\u51FA\u9519
+secretutil.4=rsa\u89E3\u5BC6\u51FA\u9519
+secretutil.5=3\u91CDDES\u52A0\u5BC6\u51FA\u9519
+secretutil.6=rsa\u89E3\u5BC6\u51FA\u9519
+secretutil.7=\u6784\u5EFA\u4E09\u91CDDES\u5BC6\u5319\u51FA\u9519
+secretutil.8=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u65E0\u6CD5\u627E\u5230\u5BC6\u94A5\u7684\u914D\u7F6E\u6587\u4EF6
+secretutil.9=\u8BFB\u53D6\u52A0\u89E3\u5BC6\u914D\u7F6E\u6587\u4EF6\u51FA\u9519
+secretutil.10=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C
+secretutil.11=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7EDF\u7EF4\u62A4\u95EE\u9898
+secretutil.12=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C
+secretutil.13=DataX\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C\u4E3A[{0}]\uFF0C\u4F46\u5728\u7CFB\u7EDF\u4E2D\u6CA1\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52A1\u5BC6\u94A5\u914D\u7F6E\u9519\u8BEF\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7EDF\u7EF4\u62A4\u95EE\u9898
+secretutil.14=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u5BC6\u94A5\u7248\u672C[{0}]\u5B58\u5728\u5BC6\u94A5\u4E3A\u7A7A\u7684\u60C5\u51B5
+secretutil.15=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u516C\u79C1\u94A5\u5BF9\u5B58\u5728\u4E3A\u7A7A\u7684\u60C5\u51B5\uFF0C\u7248\u672C[{0}]
+secretutil.16=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u65E0\u6CD5\u627E\u5230\u52A0\u89E3\u5BC6\u914D\u7F6E
+
+configparser.1=\u5916\u639B\u7A0B\u5F0F[{0},{1}]\u8F09\u5165\u5931\u6557\uFF0C1s\u5F8C\u91CD\u8A66... Exception:{2}
+configparser.2=\u7372\u53D6\u4F5C\u696D\u914D\u7F6E\u8CC7\u8A0A\u5931\u6557:{0}
+configparser.3=\u7372\u53D6\u4F5C\u696D\u914D\u7F6E\u8CC7\u8A0A\u5931\u6557:{0}
+configparser.4=\u7372\u53D6\u4F5C\u696D\u914D\u7F6E\u8CC7\u8A0A\u5931\u6557:{0}
+configparser.5=\u5916\u639B\u7A0B\u5F0F\u8F09\u5165\u5931\u6557\uFF0C\u672A\u5B8C\u6210\u6307\u5B9A\u5916\u639B\u7A0B\u5F0F\u8F09\u5165:{0}
+configparser.6=\u5916\u639B\u7A0B\u5F0F\u8F09\u5165\u5931\u6557,\u5B58\u5728\u91CD\u8907\u5916\u639B\u7A0B\u5F0F:{0}
+
+dataxserviceutil.1=\u5EFA\u7ACB\u7C3D\u540D\u7570\u5E38NoSuchAlgorithmException, [{0}]
+dataxserviceutil.2=\u5EFA\u7ACB\u7C3D\u540D\u7570\u5E38InvalidKeyException, [{0}]
+dataxserviceutil.3=\u5EFA\u7ACB\u7C3D\u540D\u7570\u5E38UnsupportedEncodingException, [{0}]
+
+errorrecordchecker.1=\u9AD2\u6578\u64DA\u767E\u5206\u6BD4\u9650\u5236\u61C9\u8A72\u5728[0.0, 1.0]\u4E4B\u9593
+errorrecordchecker.2=\u9AD2\u6578\u64DA\u689D\u6578\u73FE\u5728\u61C9\u8A72\u70BA\u975E\u8CA0\u6574\u6578
+errorrecordchecker.3=\u9AD2\u6578\u64DA\u689D\u6578\u6AA2\u67E5\u4E0D\u901A\u904E\uFF0C\u9650\u5236\u662F[{0}]\u689D\uFF0C\u4F46\u5BE6\u969B\u4E0A\u6355\u7372\u4E86[{1}]\u689D.
+errorrecordchecker.4=\u9AD2\u6578\u64DA\u767E\u5206\u6BD4\u6AA2\u67E5\u4E0D\u901A\u904E\uFF0C\u9650\u5236\u662F[{0}]\uFF0C\u4F46\u5BE6\u969B\u4E0A\u6355\u7372\u5230[{1}].
+
+
+errorcode.install_error=DataX\u5F15\u64CE\u5B89\u88DD\u932F\u8AA4, \u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.argument_error=DataX\u5F15\u64CE\u904B\u884C\u932F\u8AA4\uFF0C\u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BC\u5167\u90E8\u7DE8\u7A0B\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61DataX\u958B\u767C\u5718\u968A\u89E3\u6C7A .
+errorcode.runtime_error=DataX\u5F15\u64CE\u904B\u884C\u904E\u7A0B\u51FA\u932F\uFF0C\u5177\u9AD4\u539F\u56E0\u8ACB\u53C3\u770BDataX\u904B\u884C\u7D50\u675F\u6642\u7684\u932F\u8AA4\u8A3A\u65B7\u8CC7\u8A0A .
+errorcode.config_error=DataX\u5F15\u64CE\u914D\u7F6E\u932F\u8AA4\uFF0C\u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5B89\u88DD\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.secret_error=DataX\u5F15\u64CE\u52A0\u89E3\u5BC6\u51FA\u932F\uFF0C\u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5BC6\u9470\u914D\u7F6E\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.hook_load_error=\u8F09\u5165\u5916\u90E8Hook\u51FA\u73FE\u932F\u8AA4\uFF0C\u901A\u5E38\u662F\u7531\u65BCDataX\u5B89\u88DD\u5F15\u8D77\u7684
+errorcode.hook_fail_error=\u57F7\u884C\u5916\u90E8Hook\u51FA\u73FE\u932F\u8AA4
+errorcode.plugin_install_error=DataX\u5916\u639B\u7A0B\u5F0F\u5B89\u88DD\u932F\u8AA4, \u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5B89\u88DD\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.plugin_not_found=DataX\u5916\u639B\u7A0B\u5F0F\u914D\u7F6E\u932F\u8AA4, \u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5B89\u88DD\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.plugin_init_error=DataX\u5916\u639B\u7A0B\u5F0F\u521D\u59CB\u5316\u932F\u8AA4, \u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5B89\u88DD\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61\u60A8\u7684\u904B\u7DAD\u89E3\u6C7A .
+errorcode.plugin_runtime_error=DataX\u5916\u639B\u7A0B\u5F0F\u904B\u884C\u6642\u51FA\u932F, \u5177\u9AD4\u539F\u56E0\u8ACB\u53C3\u770BDataX\u904B\u884C\u7D50\u675F\u6642\u7684\u932F\u8AA4\u8A3A\u65B7\u8CC7\u8A0A .
+errorcode.plugin_dirty_data_limit_exceed=DataX\u50B3\u8F38\u9AD2\u6578\u64DA\u8D85\u904E\u7528\u6236\u9810\u671F\uFF0C\u8A72\u932F\u8AA4\u901A\u5E38\u662F\u7531\u65BC\u6E90\u7AEF\u6578\u64DA\u5B58\u5728\u8F03\u591A\u696D\u52D9\u9AD2\u6578\u64DA\u5C0E\u81F4\uFF0C\u8ACB\u4ED4\u7D30\u6AA2\u67E5DataX\u5F59\u5831\u7684\u9AD2\u6578\u64DA\u65E5\u8A8C\u8CC7\u8A0A, \u6216\u8005\u60A8\u53EF\u4EE5\u9069\u7576\u8ABF\u5927\u9AD2\u6578\u64DA\u95BE\u503C .
+errorcode.plugin_split_error=DataX\u5916\u639B\u7A0B\u5F0F\u5207\u5206\u51FA\u932F, \u8A72\u554F\u984C\u901A\u5E38\u662F\u7531\u65BCDataX\u5404\u500B\u5916\u639B\u7A0B\u5F0F\u7DE8\u7A0B\u932F\u8AA4\u5F15\u8D77\uFF0C\u8ACB\u806F\u7D61DataX\u958B\u767C\u5718\u968A\u89E3\u6C7A
+errorcode.kill_job_timeout_error=kill \u4EFB\u52D9\u903E\u6642\uFF0C\u8ACB\u806F\u7D61PE\u89E3\u6C7A
+errorcode.start_taskgroup_error=taskGroup\u555F\u52D5\u5931\u6557,\u8ACB\u806F\u7D61DataX\u958B\u767C\u5718\u968A\u89E3\u6C7A
+errorcode.call_datax_service_failed=\u8ACB\u6C42 DataX Service \u51FA\u932F.
+errorcode.call_remote_failed=\u9060\u7A0B\u8ABF\u7528\u5931\u6557
+errorcode.killed_exit_value=Job \u6536\u5230\u4E86 Kill \u547D\u4EE4.
+
+
+httpclientutil.1=\u8ACB\u6C42\u5730\u5740\uFF1A{0}, \u8ACB\u6C42\u65B9\u6CD5\uFF1A{1},STATUS CODE = {2}, Response Entity: {3}
+httpclientutil.2=\u9060\u7A0B\u63A5\u53E3\u8FD4\u56DE-1,\u5C07\u91CD\u8A66
+
+
+secretutil.1=\u7CFB\u7D71\u7DE8\u7A0B\u932F\u8AA4,\u4E0D\u652F\u63F4\u7684\u52A0\u5BC6\u985E\u578B
+secretutil.2=\u7CFB\u7D71\u7DE8\u7A0B\u932F\u8AA4,\u4E0D\u652F\u63F4\u7684\u52A0\u5BC6\u985E\u578B
+secretutil.3=rsa\u52A0\u5BC6\u51FA\u932F
+secretutil.4=rsa\u89E3\u5BC6\u51FA\u932F
+secretutil.5=3\u91CDDES\u52A0\u5BC6\u51FA\u932F
+secretutil.6=rsa\u89E3\u5BC6\u51FA\u932F
+secretutil.7=\u69CB\u5EFA\u4E09\u91CDDES\u5BC6\u5319\u51FA\u932F
+secretutil.8=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u7121\u6CD5\u627E\u5230\u5BC6\u9470\u7684\u914D\u7F6E\u6A94\u6848
+secretutil.9=\u8B80\u53D6\u52A0\u89E3\u5BC6\u914D\u7F6E\u6A94\u6848\u51FA\u932F
+secretutil.10=DataX\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C\u70BA[{0}]\uFF0C\u4F46\u5728\u7CFB\u7D71\u4E2D\u6C92\u6709\u914D\u7F6E\uFF0C\u4EFB\u52D9\u5BC6\u9470\u914D\u7F6E\u932F\u8AA4\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C
+secretutil.11=DataX\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C\u70BA[{0}]\uFF0C\u4F46\u5728\u7CFB\u7D71\u4E2D\u6C92\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52D9\u5BC6\u9470\u914D\u7F6E\u932F\u8AA4\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7D71\u7DAD\u8B77\u554F\u984C
+secretutil.12=DataX\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C\u70BA[{0}]\uFF0C\u4F46\u5728\u7CFB\u7D71\u4E2D\u6C92\u6709\u914D\u7F6E\uFF0C\u4EFB\u52D9\u5BC6\u9470\u914D\u7F6E\u932F\u8AA4\uFF0C\u4E0D\u5B58\u5728\u60A8\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C
+secretutil.13=DataX\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C\u70BA[{0}]\uFF0C\u4F46\u5728\u7CFB\u7D71\u4E2D\u6C92\u6709\u914D\u7F6E\uFF0C\u53EF\u80FD\u662F\u4EFB\u52D9\u5BC6\u9470\u914D\u7F6E\u932F\u8AA4\uFF0C\u4E5F\u53EF\u80FD\u662F\u7CFB\u7D71\u7DAD\u8B77\u554F\u984C
+secretutil.14=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u5BC6\u9470\u7248\u672C[{0}]\u5B58\u5728\u5BC6\u9470\u70BA\u7A7A\u7684\u60C5\u6CC1
+secretutil.15=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u914D\u7F6E\u7684\u516C\u79C1\u9470\u5C0D\u5B58\u5728\u70BA\u7A7A\u7684\u60C5\u6CC1\uFF0C\u7248\u672C[{0}]
+secretutil.16=DataX\u914D\u7F6E\u8981\u6C42\u52A0\u89E3\u5BC6\uFF0C\u4F46\u7121\u6CD5\u627E\u5230\u52A0\u89E3\u5BC6\u914D\u7F6E
+
diff --git a/core/src/main/job/job.json b/core/src/main/job/job.json
index 58206592..cc353877 100755
--- a/core/src/main/job/job.json
+++ b/core/src/main/job/job.json
@@ -2,7 +2,7 @@
"job": {
"setting": {
"speed": {
- "byte":10485760
+ "channel":1
},
"errorLimit": {
"record": 0,
diff --git a/databendwriter/doc/databendwriter-CN.md b/databendwriter/doc/databendwriter-CN.md
new file mode 100644
index 00000000..d6a8f1f3
--- /dev/null
+++ b/databendwriter/doc/databendwriter-CN.md
@@ -0,0 +1,171 @@
+# DataX DatabendWriter
+[简体中文](./databendwriter-CN.md) | [English](./databendwriter.md)
+
+## 1 快速介绍
+
+Databend Writer 是一个 DataX 的插件，用于从 DataX 中写入数据到 Databend 表中。
+该插件基于[databend JDBC driver](https://github.com/databendcloud/databend-jdbc) ，它使用 [RESTful http protocol](https://databend.rs/doc/integrations/api/rest)
+在开源的 databend 和 [databend cloud](https://app.databend.com/) 上执行查询。
+
+在每个写入批次中，databend writer 将批量数据上传到内部的 S3 stage，然后执行相应的 insert SQL 将数据上传到 databend 表中。
+
+为了最佳的用户体验，如果您使用的是 databend 社区版本，您应该尝试采用 [S3](https://aws.amazon.com/s3/)/[minio](https://min.io/)/[OSS](https://www.alibabacloud.com/product/object-storage-service) 作为其底层存储层，因为
+它们支持预签名上传操作，否则您可能会在数据传输上浪费不必要的成本。
+
+您可以在[文档](https://databend.rs/doc/deploy/deploying-databend)中了解更多详细信息
+
+## 2 实现原理
+
+Databend Writer 将使用 DataX 从 DataX Reader 中获取生成的记录，并将记录批量插入到 databend 表中指定的列中。
+
+## 3 功能说明
+
+### 3.1 配置样例
+
+* 以下配置将从内存中读取一些生成的数据，并将数据上传到databend表中
+
+#### 准备工作
+```sql
+--- create table in databend
+drop table if exists datax.sample1;
+drop database if exists datax;
+create database if not exists datax;
+create table if not exsits datax.sample1(a string, b int64, c date, d timestamp, e bool, f string, g variant);
+```
+
+#### 配置样例
+```json
+{
+ "job": {
+ "content": [
+ {
+ "reader": {
+ "name": "streamreader",
+ "parameter": {
+ "column" : [
+ {
+ "value": "DataX",
+ "type": "string"
+ },
+ {
+ "value": 19880808,
+ "type": "long"
+ },
+ {
+ "value": "1926-08-08 08:08:08",
+ "type": "date"
+ },
+ {
+ "value": "1988-08-08 08:08:08",
+ "type": "date"
+ },
+ {
+ "value": true,
+ "type": "bool"
+ },
+ {
+ "value": "test",
+ "type": "bytes"
+ },
+ {
+ "value": "{\"type\": \"variant\", \"value\": \"test\"}",
+ "type": "string"
+ }
+
+ ],
+ "sliceRecordCount": 10000
+ }
+ },
+ "writer": {
+ "name": "databendwriter",
+ "parameter": {
+ "username": "databend",
+ "password": "databend",
+ "column": ["a", "b", "c", "d", "e", "f", "g"],
+ "batchSize": 1000,
+ "preSql": [
+ ],
+ "postSql": [
+ ],
+ "connection": [
+ {
+ "jdbcUrl": "jdbc:databend://localhost:8000/datax",
+ "table": [
+ "sample1"
+ ]
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "setting": {
+ "speed": {
+ "channel": 1
+ }
+ }
+ }
+}
+```
+
+### 3.2 参数说明
+* jdbcUrl
+ * 描述: JDBC 数据源 url。请参阅仓库中的详细[文档](https://github.com/databendcloud/databend-jdbc)
+ * 必选: 是
+ * 默认值: 无
+ * 示例: jdbc:databend://localhost:8000/datax
+* username
+ * 描述: JDBC 数据源用户名
+ * 必选: 是
+ * 默认值: 无
+ * 示例: databend
+* password
+ * 描述: JDBC 数据源密码
+ * 必选: 是
+ * 默认值: 无
+ * 示例: databend
+* table
+ * 描述: 表名的集合，table应该包含column参数中的所有列。
+ * 必选: 是
+ * 默认值: 无
+ * 示例: ["sample1"]
+* column
+ * 描述: 表中的列名集合，字段顺序应该与reader的record中的column类型对应
+ * 必选: 是
+ * 默认值: 无
+ * 示例: ["a", "b", "c", "d", "e", "f", "g"]
+* batchSize
+ * 描述: 每个批次的记录数
+ * 必选: 否
+ * 默认值: 1000
+ * 示例: 1000
+* preSql
+ * 描述: 在写入数据之前执行的SQL语句
+ * 必选: 否
+ * 默认值: 无
+ * 示例: ["delete from datax.sample1"]
+* postSql
+ * 描述: 在写入数据之后执行的SQL语句
+ * 必选: 否
+ * 默认值: 无
+ * 示例: ["select count(*) from datax.sample1"]
+
+### 3.3 类型转化
+DataX中的数据类型可以转换为databend中的相应数据类型。下表显示了两种类型之间的对应关系。
+
+| DataX 内部类型 | Databend 数据类型 |
+|------------|-----------------------------------------------------------|
+| INT | TINYINT, INT8, SMALLINT, INT16, INT, INT32, BIGINT, INT64 |
+| LONG | TINYINT, INT8, SMALLINT, INT16, INT, INT32, BIGINT, INT64 |
+| STRING | STRING, VARCHAR |
+| DOUBLE | FLOAT, DOUBLE |
+| BOOL | BOOLEAN, BOOL |
+| DATE | DATE, TIMESTAMP |
+| BYTES | STRING, VARCHAR |
+
+## 4 性能测试
+
+## 5 约束限制
+目前，复杂数据类型支持不稳定，如果您想使用复杂数据类型，例如元组，数组，请检查databend和jdbc驱动程序的进一步版本。
+
+## FAQ
\ No newline at end of file
diff --git a/databendwriter/doc/databendwriter.md b/databendwriter/doc/databendwriter.md
new file mode 100644
index 00000000..0b57bf13
--- /dev/null
+++ b/databendwriter/doc/databendwriter.md
@@ -0,0 +1,166 @@
+# DataX DatabendWriter
+[简体中文](./databendwriter-CN.md) | [English](./databendwriter.md)
+
+## 1 Introduction
+Databend Writer is a plugin for DataX to write data to Databend Table from dataX records.
+The plugin is based on [databend JDBC driver](https://github.com/databendcloud/databend-jdbc) which use [RESTful http protocol](https://databend.rs/doc/integrations/api/rest)
+to execute query on open source databend and [databend cloud](https://app.databend.com/).
+
+During each write batch, databend writer will upload batch data into internal S3 stage and execute corresponding insert SQL to upload data into databend table.
+
+For best user experience, if you are using databend community distribution, you should try to adopt [S3](https://aws.amazon.com/s3/)/[minio](https://min.io/)/[OSS](https://www.alibabacloud.com/product/object-storage-service) as its underlying storage layer since
+they support presign upload operation otherwise you may expend unneeded cost on data transfer.
+
+You could see more details on the [doc](https://databend.rs/doc/deploy/deploying-databend)
+
+## 2 Detailed Implementation
+Databend Writer would use DataX to fetch records generated by DataX Reader, and then batch insert records to the designated columns for your databend table.
+
+## 3 Features
+### 3.1 Example Configurations
+* the following configuration would read some generated data in memory and upload data into databend table
+
+#### Preparation
+```sql
+--- create table in databend
+drop table if exists datax.sample1;
+drop database if exists datax;
+create database if not exists datax;
+create table if not exsits datax.sample1(a string, b int64, c date, d timestamp, e bool, f string, g variant);
+```
+
+#### Configurations
+```json
+{
+ "job": {
+ "content": [
+ {
+ "reader": {
+ "name": "streamreader",
+ "parameter": {
+ "column" : [
+ {
+ "value": "DataX",
+ "type": "string"
+ },
+ {
+ "value": 19880808,
+ "type": "long"
+ },
+ {
+ "value": "1926-08-08 08:08:08",
+ "type": "date"
+ },
+ {
+ "value": "1988-08-08 08:08:08",
+ "type": "date"
+ },
+ {
+ "value": true,
+ "type": "bool"
+ },
+ {
+ "value": "test",
+ "type": "bytes"
+ },
+ {
+ "value": "{\"type\": \"variant\", \"value\": \"test\"}",
+ "type": "string"
+ }
+
+ ],
+ "sliceRecordCount": 10000
+ }
+ },
+ "writer": {
+ "name": "databendwriter",
+ "parameter": {
+ "username": "databend",
+ "password": "databend",
+ "column": ["a", "b", "c", "d", "e", "f", "g"],
+ "batchSize": 1000,
+ "preSql": [
+ ],
+ "postSql": [
+ ],
+ "connection": [
+ {
+ "jdbcUrl": "jdbc:databend://localhost:8000/datax",
+ "table": [
+ "sample1"
+ ]
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "setting": {
+ "speed": {
+ "channel": 1
+ }
+ }
+ }
+}
+```
+
+### 3.2 Configuration Description
+* jdbcUrl
+ * Description: JDBC Data source url in Databend. Please take a look at repository for detailed [doc](https://github.com/databendcloud/databend-jdbc)
+ * Required: yes
+ * Default: none
+ * Example: jdbc:databend://localhost:8000/datax
+* username
+ * Description: Databend user name
+ * Required: yes
+ * Default: none
+ * Example: databend
+* password
+ * Description: Databend user password
+ * Required: yes
+ * Default: none
+ * Example: databend
+* table
+ * Description: A list of table names that should contain all of the columns in the column parameter.
+ * Required: yes
+ * Default: none
+ * Example: ["sample1"]
+* column
+ * Description: A list of column field names that should be inserted into the table. if you want to insert all column fields use `["*"]` instead.
+ * Required: yes
+ * Default: none
+ * Example: ["a", "b", "c", "d", "e", "f", "g"]
+* batchSize
+ * Description: The number of records to be inserted in each batch.
+ * Required: no
+ * Default: 1024
+* preSql
+ * Description: A list of SQL statements that will be executed before the write operation.
+ * Required: no
+ * Default: none
+* postSql
+ * Description: A list of SQL statements that will be executed after the write operation.
+ * Required: no
+ * Default: none
+
+### 3.3 Type Convert
+Data types in datax can be converted to the corresponding data types in databend. The following table shows the correspondence between the two types.
+
+| DataX Type | Databend Type |
+|------------|-----------------------------------------------------------|
+| INT | TINYINT, INT8, SMALLINT, INT16, INT, INT32, BIGINT, INT64 |
+| LONG | TINYINT, INT8, SMALLINT, INT16, INT, INT32, BIGINT, INT64 |
+| STRING | STRING, VARCHAR |
+| DOUBLE | FLOAT, DOUBLE |
+| BOOL | BOOLEAN, BOOL |
+| DATE | DATE, TIMESTAMP |
+| BYTES | STRING, VARCHAR |
+
+
+## 4 Performance Test
+
+
+## 5 Restrictions
+Currently, complex data type support is not stable, if you want to use complex data type such as tuple, array, please check further release version of databend and jdbc driver.
+
+## FAQ
diff --git a/databendwriter/pom.xml b/databendwriter/pom.xml
new file mode 100644
index 00000000..976ecd6a
--- /dev/null
+++ b/databendwriter/pom.xml
@@ -0,0 +1,101 @@
+
+
+
+ datax-all
+ com.alibaba.datax
+ 0.0.1-SNAPSHOT
+
+
+ 4.0.0
+ databendwriter
+ databendwriter
+ jar
+
+
+
+ com.databend
+ databend-jdbc
+ 0.0.5
+
+
+ com.alibaba.datax
+ datax-core
+ ${datax-project-version}
+
+
+ com.alibaba.datax
+ datax-common
+ ${datax-project-version}
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+ ch.qos.logback
+ logback-classic
+
+
+
+ com.alibaba.datax
+ plugin-rdbms-util
+ ${datax-project-version}
+
+
+ com.google.guava
+ guava
+
+
+
+
+
+
+ junit
+ junit
+ test
+
+
+
+
+
+ src/main/java
+
+ **/*.properties
+
+
+
+
+
+
+ maven-compiler-plugin
+
+ ${jdk-version}
+ ${jdk-version}
+ ${project-sourceEncoding}
+
+
+
+
+ maven-assembly-plugin
+
+
+ src/main/assembly/package.xml
+
+ datax
+
+
+
+ dwzip
+ package
+
+ single
+
+
+
+
+
+
+
diff --git a/databendwriter/src/main/assembly/package.xml b/databendwriter/src/main/assembly/package.xml
new file mode 100755
index 00000000..8a9ba1b2
--- /dev/null
+++ b/databendwriter/src/main/assembly/package.xml
@@ -0,0 +1,34 @@
+
+
+
+ dir
+
+ false
+
+
+ src/main/resources
+
+ plugin.json
+ plugin_job_template.json
+
+ plugin/writer/databendwriter
+
+
+ target/
+
+ databendwriter-0.0.1-SNAPSHOT.jar
+
+ plugin/writer/databendwriter
+
+
+
+
+
+ false
+ plugin/writer/databendwriter/libs
+
+
+
diff --git a/databendwriter/src/main/java/com/alibaba/datax/plugin/writer/databendwriter/DatabendWriter.java b/databendwriter/src/main/java/com/alibaba/datax/plugin/writer/databendwriter/DatabendWriter.java
new file mode 100644
index 00000000..a4222f08
--- /dev/null
+++ b/databendwriter/src/main/java/com/alibaba/datax/plugin/writer/databendwriter/DatabendWriter.java
@@ -0,0 +1,248 @@
+package com.alibaba.datax.plugin.writer.databendwriter;
+
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.element.StringColumn;
+import com.alibaba.datax.common.exception.CommonErrorCode;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+import com.alibaba.datax.common.spi.Writer;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
+import com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter;
+import com.alibaba.datax.plugin.writer.databendwriter.util.DatabendWriterUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.*;
+import java.util.List;
+import java.util.regex.Pattern;
+
+public class DatabendWriter extends Writer
+{
+ private static final DataBaseType DATABASE_TYPE = DataBaseType.Databend;
+
+ public static class Job
+ extends Writer.Job
+ {
+ private static final Logger LOG = LoggerFactory.getLogger(Job.class);
+ private Configuration originalConfig;
+ private CommonRdbmsWriter.Job commonRdbmsWriterMaster;
+
+ @Override
+ public void init()
+ {
+ this.originalConfig = super.getPluginJobConf();
+ this.commonRdbmsWriterMaster = new CommonRdbmsWriter.Job(DATABASE_TYPE);
+ this.commonRdbmsWriterMaster.init(this.originalConfig);
+ // placeholder currently not supported by databend driver, needs special treatment
+ DatabendWriterUtil.dealWriteMode(this.originalConfig);
+ }
+
+ @Override
+ public void preCheck()
+ {
+ this.init();
+ this.commonRdbmsWriterMaster.writerPreCheck(this.originalConfig, DATABASE_TYPE);
+ }
+
+ @Override
+ public void prepare() {
+ this.commonRdbmsWriterMaster.prepare(this.originalConfig);
+ }
+
+ @Override
+ public List split(int mandatoryNumber) {
+ return this.commonRdbmsWriterMaster.split(this.originalConfig, mandatoryNumber);
+ }
+
+ @Override
+ public void post() {
+ this.commonRdbmsWriterMaster.post(this.originalConfig);
+ }
+
+ @Override
+ public void destroy() {
+ this.commonRdbmsWriterMaster.destroy(this.originalConfig);
+ }
+ }
+
+
+ public static class Task extends Writer.Task
+ {
+ private static final Logger LOG = LoggerFactory.getLogger(Task.class);
+
+ private Configuration writerSliceConfig;
+
+ private CommonRdbmsWriter.Task commonRdbmsWriterSlave;
+
+ @Override
+ public void init()
+ {
+ this.writerSliceConfig = super.getPluginJobConf();
+
+ this.commonRdbmsWriterSlave = new CommonRdbmsWriter.Task(DataBaseType.Databend){
+ @Override
+ protected PreparedStatement fillPreparedStatementColumnType(PreparedStatement preparedStatement, int columnIndex, int columnSqltype, String typeName, Column column) throws SQLException {
+ try {
+ if (column.getRawData() == null) {
+ preparedStatement.setNull(columnIndex + 1, columnSqltype);
+ return preparedStatement;
+ }
+
+ java.util.Date utilDate;
+ switch (columnSqltype) {
+
+ case Types.TINYINT:
+ case Types.SMALLINT:
+ case Types.INTEGER:
+ preparedStatement.setInt(columnIndex + 1, column.asBigInteger().intValue());
+ break;
+ case Types.BIGINT:
+ preparedStatement.setLong(columnIndex + 1, column.asLong());
+ break;
+ case Types.DECIMAL:
+ preparedStatement.setBigDecimal(columnIndex + 1, column.asBigDecimal());
+ break;
+ case Types.FLOAT:
+ case Types.REAL:
+ preparedStatement.setFloat(columnIndex + 1, column.asDouble().floatValue());
+ break;
+ case Types.DOUBLE:
+ preparedStatement.setDouble(columnIndex + 1, column.asDouble());
+ break;
+ case Types.DATE:
+ java.sql.Date sqlDate = null;
+ try {
+ utilDate = column.asDate();
+ } catch (DataXException e) {
+ throw new SQLException(String.format(
+ "Date type conversion error: [%s]", column));
+ }
+
+ if (null != utilDate) {
+ sqlDate = new java.sql.Date(utilDate.getTime());
+ }
+ preparedStatement.setDate(columnIndex + 1, sqlDate);
+ break;
+
+ case Types.TIME:
+ java.sql.Time sqlTime = null;
+ try {
+ utilDate = column.asDate();
+ } catch (DataXException e) {
+ throw new SQLException(String.format(
+ "Date type conversion error: [%s]", column));
+ }
+
+ if (null != utilDate) {
+ sqlTime = new java.sql.Time(utilDate.getTime());
+ }
+ preparedStatement.setTime(columnIndex + 1, sqlTime);
+ break;
+
+ case Types.TIMESTAMP:
+ Timestamp sqlTimestamp = null;
+ if (column instanceof StringColumn && column.asString() != null) {
+ String timeStampStr = column.asString();
+ // JAVA TIMESTAMP 类型入参必须是 "2017-07-12 14:39:00.123566" 格式
+ String pattern = "^\\d+-\\d+-\\d+ \\d+:\\d+:\\d+.\\d+";
+ boolean isMatch = Pattern.matches(pattern, timeStampStr);
+ if (isMatch) {
+ sqlTimestamp = Timestamp.valueOf(timeStampStr);
+ preparedStatement.setTimestamp(columnIndex + 1, sqlTimestamp);
+ break;
+ }
+ }
+ try {
+ utilDate = column.asDate();
+ } catch (DataXException e) {
+ throw new SQLException(String.format(
+ "Date type conversion error: [%s]", column));
+ }
+
+ if (null != utilDate) {
+ sqlTimestamp = new Timestamp(
+ utilDate.getTime());
+ }
+ preparedStatement.setTimestamp(columnIndex + 1, sqlTimestamp);
+ break;
+
+ case Types.BINARY:
+ case Types.VARBINARY:
+ case Types.BLOB:
+ case Types.LONGVARBINARY:
+ preparedStatement.setBytes(columnIndex + 1, column
+ .asBytes());
+ break;
+
+ case Types.BOOLEAN:
+
+ // warn: bit(1) -> Types.BIT 可使用setBoolean
+ // warn: bit(>1) -> Types.VARBINARY 可使用setBytes
+ case Types.BIT:
+ if (this.dataBaseType == DataBaseType.MySql) {
+ Boolean asBoolean = column.asBoolean();
+ if (asBoolean != null) {
+ preparedStatement.setBoolean(columnIndex + 1, asBoolean);
+ } else {
+ preparedStatement.setNull(columnIndex + 1, Types.BIT);
+ }
+ } else {
+ preparedStatement.setString(columnIndex + 1, column.asString());
+ }
+ break;
+
+ default:
+ // cast variant / array into string is fine.
+ preparedStatement.setString(columnIndex + 1, column.asString());
+ break;
+ }
+ return preparedStatement;
+ } catch (DataXException e) {
+ // fix类型转换或者溢出失败时，将具体哪一列打印出来
+ if (e.getErrorCode() == CommonErrorCode.CONVERT_NOT_SUPPORT ||
+ e.getErrorCode() == CommonErrorCode.CONVERT_OVER_FLOW) {
+ throw DataXException
+ .asDataXException(
+ e.getErrorCode(),
+ String.format(
+ "type conversion error. columnName: [%s], columnType:[%d], columnJavaType: [%s]. please change the data type in given column field or do not sync on the column.",
+ this.resultSetMetaData.getLeft()
+ .get(columnIndex),
+ this.resultSetMetaData.getMiddle()
+ .get(columnIndex),
+ this.resultSetMetaData.getRight()
+ .get(columnIndex)));
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ };
+ this.commonRdbmsWriterSlave.init(this.writerSliceConfig);
+ }
+
+ @Override
+ public void destroy()
+ {
+ this.commonRdbmsWriterSlave.destroy(this.writerSliceConfig);
+ }
+
+ @Override
+ public void prepare() {
+ this.commonRdbmsWriterSlave.prepare(this.writerSliceConfig);
+ }
+
+ @Override
+ public void post() {
+ this.commonRdbmsWriterSlave.post(this.writerSliceConfig);
+ }
+ @Override
+ public void startWrite(RecordReceiver lineReceiver)
+ {
+ this.commonRdbmsWriterSlave.startWrite(lineReceiver, this.writerSliceConfig, this.getTaskPluginCollector());
+ }
+
+ }
+}
diff --git a/databendwriter/src/main/java/com/alibaba/datax/plugin/writer/databendwriter/util/DatabendWriterUtil.java b/databendwriter/src/main/java/com/alibaba/datax/plugin/writer/databendwriter/util/DatabendWriterUtil.java
new file mode 100644
index 00000000..a862e920
--- /dev/null
+++ b/databendwriter/src/main/java/com/alibaba/datax/plugin/writer/databendwriter/util/DatabendWriterUtil.java
@@ -0,0 +1,40 @@
+package com.alibaba.datax.plugin.writer.databendwriter.util;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.rdbms.writer.Constant;
+import com.alibaba.datax.plugin.rdbms.writer.Key;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.StringJoiner;
+
+public final class DatabendWriterUtil
+{
+ private static final Logger LOG = LoggerFactory.getLogger(DatabendWriterUtil.class);
+
+ private DatabendWriterUtil() {}
+ public static void dealWriteMode(Configuration originalConfig)
+ {
+ List columns = originalConfig.getList(Key.COLUMN, String.class);
+
+ String jdbcUrl = originalConfig.getString(String.format("%s[0].%s",
+ Constant.CONN_MARK, Key.JDBC_URL, String.class));
+
+ String writeMode = originalConfig.getString(Key.WRITE_MODE, "INSERT");
+
+ StringBuilder writeDataSqlTemplate = new StringBuilder();
+ writeDataSqlTemplate.append("INSERT INTO %s");
+ StringJoiner columnString = new StringJoiner(",");
+
+ for (String column : columns) {
+ columnString.add(column);
+ }
+ writeDataSqlTemplate.append(String.format("(%s)", columnString));
+ writeDataSqlTemplate.append(" VALUES");
+
+ LOG.info("Write data [\n{}\n], which jdbcUrl like:[{}]", writeDataSqlTemplate, jdbcUrl);
+
+ originalConfig.set(Constant.INSERT_OR_REPLACE_TEMPLATE_MARK, writeDataSqlTemplate);
+ }
+}
\ No newline at end of file
diff --git a/databendwriter/src/main/resources/plugin.json b/databendwriter/src/main/resources/plugin.json
new file mode 100644
index 00000000..bab0130d
--- /dev/null
+++ b/databendwriter/src/main/resources/plugin.json
@@ -0,0 +1,6 @@
+{
+ "name": "databendwriter",
+ "class": "com.alibaba.datax.plugin.writer.databendwriter.DatabendWriter",
+ "description": "execute batch insert sql to write dataX data into databend",
+ "developer": "databend"
+}
\ No newline at end of file
diff --git a/databendwriter/src/main/resources/plugin_job_template.json b/databendwriter/src/main/resources/plugin_job_template.json
new file mode 100644
index 00000000..34d4b251
--- /dev/null
+++ b/databendwriter/src/main/resources/plugin_job_template.json
@@ -0,0 +1,19 @@
+{
+ "name": "databendwriter",
+ "parameter": {
+ "username": "username",
+ "password": "password",
+ "column": ["col1", "col2", "col3"],
+ "connection": [
+ {
+ "jdbcUrl": "jdbc:databend://:[/]",
+ "table": "table1"
+ }
+ ],
+ "preSql": [],
+ "postSql": [],
+
+ "maxBatchRows": 65536,
+ "maxBatchSize": 134217728
+ }
+}
\ No newline at end of file
diff --git a/datahubreader/pom.xml b/datahubreader/pom.xml
new file mode 100644
index 00000000..c0022b44
--- /dev/null
+++ b/datahubreader/pom.xml
@@ -0,0 +1,79 @@
+
+
+
+ datax-all
+ com.alibaba.datax
+ 0.0.1-SNAPSHOT
+
+ 4.0.0
+
+ datahubreader
+
+ 0.0.1-SNAPSHOT
+
+
+
+ com.alibaba.datax
+ datax-common
+ ${datax-project-version}
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+ ch.qos.logback
+ logback-classic
+
+
+ com.aliyun.datahub
+ aliyun-sdk-datahub
+ 2.21.6-public
+
+
+ junit
+ junit
+ 4.12
+ test
+
+
+
+
+
+
+
+ maven-compiler-plugin
+
+ ${jdk-version}
+ ${jdk-version}
+ ${project-sourceEncoding}
+
+
+
+
+ maven-assembly-plugin
+
+
+ src/main/assembly/package.xml
+
+ datax
+
+
+
+ dwzip
+ package
+
+ single
+
+
+
+
+
+
+
diff --git a/datahubreader/src/main/assembly/package.xml b/datahubreader/src/main/assembly/package.xml
new file mode 100644
index 00000000..d14ea981
--- /dev/null
+++ b/datahubreader/src/main/assembly/package.xml
@@ -0,0 +1,34 @@
+
+
+
+ dir
+
+ false
+
+
+ src/main/resources
+
+ plugin.json
+
+ plugin/reader/datahubreader
+
+
+ target/
+
+ datahubreader-0.0.1-SNAPSHOT.jar
+
+ plugin/reader/datahubreader
+
+
+
+
+
+ false
+ plugin/reader/datahubreader/libs
+ runtime
+
+
+
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Constant.java b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Constant.java
new file mode 100644
index 00000000..bee3ccd7
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Constant.java
@@ -0,0 +1,8 @@
+package com.alibaba.datax.plugin.reader.datahubreader;
+
+public class Constant {
+
+ public static String DATETIME_FORMAT = "yyyyMMddHHmmss";
+ public static String DATE_FORMAT = "yyyyMMdd";
+
+}
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubClientHelper.java b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubClientHelper.java
new file mode 100644
index 00000000..2b7bcec4
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubClientHelper.java
@@ -0,0 +1,42 @@
+package com.alibaba.datax.plugin.reader.datahubreader;
+
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.TypeReference;
+import com.aliyun.datahub.client.DatahubClient;
+import com.aliyun.datahub.client.DatahubClientBuilder;
+import com.aliyun.datahub.client.auth.Account;
+import com.aliyun.datahub.client.auth.AliyunAccount;
+import com.aliyun.datahub.client.common.DatahubConfig;
+import com.aliyun.datahub.client.http.HttpConfig;
+import org.apache.commons.lang3.StringUtils;
+
+public class DatahubClientHelper {
+ public static DatahubClient getDatahubClient(Configuration jobConfig) {
+ String accessId = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ACCESS_ID,
+ DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ String accessKey = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ACCESS_KEY,
+ DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ String endpoint = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ENDPOINT,
+ DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ Account account = new AliyunAccount(accessId, accessKey);
+ // 是否开启二进制传输，服务端2.12版本开始支持
+ boolean enableBinary = jobConfig.getBool("enableBinary", false);
+ DatahubConfig datahubConfig = new DatahubConfig(endpoint, account, enableBinary);
+ // HttpConfig可不设置，不设置时采用默认值
+ // 读写数据推荐打开网络传输 LZ4压缩
+ HttpConfig httpConfig = null;
+ String httpConfigStr = jobConfig.getString("httpConfig");
+ if (StringUtils.isNotBlank(httpConfigStr)) {
+ httpConfig = JSON.parseObject(httpConfigStr, new TypeReference() {
+ });
+ }
+
+ DatahubClientBuilder builder = DatahubClientBuilder.newBuilder().setDatahubConfig(datahubConfig);
+ if (null != httpConfig) {
+ builder.setHttpConfig(httpConfig);
+ }
+ DatahubClient datahubClient = builder.build();
+ return datahubClient;
+ }
+}
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReader.java b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReader.java
new file mode 100644
index 00000000..4792ac39
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReader.java
@@ -0,0 +1,292 @@
+package com.alibaba.datax.plugin.reader.datahubreader;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import com.aliyun.datahub.client.model.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.element.StringColumn;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordSender;
+import com.alibaba.datax.common.spi.Reader;
+import com.alibaba.datax.common.util.Configuration;
+
+
+import com.aliyun.datahub.client.DatahubClient;
+
+
+public class DatahubReader extends Reader {
+ public static class Job extends Reader.Job {
+ private static final Logger LOG = LoggerFactory.getLogger(Job.class);
+
+ private Configuration originalConfig;
+
+ private Long beginTimestampMillis;
+ private Long endTimestampMillis;
+
+ DatahubClient datahubClient;
+
+ @Override
+ public void init() {
+ LOG.info("datahub reader job init begin ...");
+ this.originalConfig = super.getPluginJobConf();
+ validateParameter(originalConfig);
+ this.datahubClient = DatahubClientHelper.getDatahubClient(this.originalConfig);
+ LOG.info("datahub reader job init end.");
+ }
+
+ private void validateParameter(Configuration conf){
+ conf.getNecessaryValue(Key.ENDPOINT,DatahubReaderErrorCode.REQUIRE_VALUE);
+ conf.getNecessaryValue(Key.ACCESSKEYID,DatahubReaderErrorCode.REQUIRE_VALUE);
+ conf.getNecessaryValue(Key.ACCESSKEYSECRET,DatahubReaderErrorCode.REQUIRE_VALUE);
+ conf.getNecessaryValue(Key.PROJECT,DatahubReaderErrorCode.REQUIRE_VALUE);
+ conf.getNecessaryValue(Key.TOPIC,DatahubReaderErrorCode.REQUIRE_VALUE);
+ conf.getNecessaryValue(Key.COLUMN,DatahubReaderErrorCode.REQUIRE_VALUE);
+ conf.getNecessaryValue(Key.BEGINDATETIME,DatahubReaderErrorCode.REQUIRE_VALUE);
+ conf.getNecessaryValue(Key.ENDDATETIME,DatahubReaderErrorCode.REQUIRE_VALUE);
+
+ int batchSize = this.originalConfig.getInt(Key.BATCHSIZE, 1024);
+ if (batchSize > 10000) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "Invalid batchSize[" + batchSize + "] value (0,10000]!");
+ }
+
+ String beginDateTime = this.originalConfig.getString(Key.BEGINDATETIME);
+ if (beginDateTime != null) {
+ try {
+ beginTimestampMillis = DatahubReaderUtils.getUnixTimeFromDateTime(beginDateTime);
+ } catch (ParseException e) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "Invalid beginDateTime[" + beginDateTime + "], format [yyyyMMddHHmmss]!");
+ }
+ }
+
+ if (beginTimestampMillis != null && beginTimestampMillis <= 0) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "Invalid beginTimestampMillis[" + beginTimestampMillis + "]!");
+ }
+
+ String endDateTime = this.originalConfig.getString(Key.ENDDATETIME);
+ if (endDateTime != null) {
+ try {
+ endTimestampMillis = DatahubReaderUtils.getUnixTimeFromDateTime(endDateTime);
+ } catch (ParseException e) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "Invalid beginDateTime[" + endDateTime + "], format [yyyyMMddHHmmss]!");
+ }
+ }
+
+ if (endTimestampMillis != null && endTimestampMillis <= 0) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "Invalid endTimestampMillis[" + endTimestampMillis + "]!");
+ }
+
+ if (beginTimestampMillis != null && endTimestampMillis != null
+ && endTimestampMillis <= beginTimestampMillis) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "endTimestampMillis[" + endTimestampMillis + "] must bigger than beginTimestampMillis[" + beginTimestampMillis + "]!");
+ }
+ }
+
+ @Override
+ public void prepare() {
+ // create datahub client
+ String project = originalConfig.getNecessaryValue(Key.PROJECT, DatahubReaderErrorCode.REQUIRE_VALUE);
+ String topic = originalConfig.getNecessaryValue(Key.TOPIC, DatahubReaderErrorCode.REQUIRE_VALUE);
+ RecordType recordType = null;
+ try {
+ DatahubClient client = DatahubClientHelper.getDatahubClient(this.originalConfig);
+ GetTopicResult getTopicResult = client.getTopic(project, topic);
+ recordType = getTopicResult.getRecordType();
+ } catch (Exception e) {
+ LOG.warn("get topic type error: {}", e.getMessage());
+ }
+ if (null != recordType) {
+ if (recordType == RecordType.BLOB) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "DatahubReader only support 'Tuple' RecordType now, but your RecordType is 'BLOB'");
+ }
+ }
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+ @Override
+ public List split(int adviceNumber) {
+ LOG.info("split() begin...");
+
+ List readerSplitConfigs = new ArrayList();
+
+ String project = this.originalConfig.getString(Key.PROJECT);
+ String topic = this.originalConfig.getString(Key.TOPIC);
+
+ List shardEntrys = DatahubReaderUtils.getShardsWithRetry(this.datahubClient, project, topic);
+ if (shardEntrys == null || shardEntrys.isEmpty() || shardEntrys.size() == 0) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "Project [" + project + "] Topic [" + topic + "] has no shards, please check !");
+ }
+
+ for (ShardEntry shardEntry : shardEntrys) {
+ Configuration splitedConfig = this.originalConfig.clone();
+ splitedConfig.set(Key.SHARDID, shardEntry.getShardId());
+ readerSplitConfigs.add(splitedConfig);
+ }
+
+ LOG.info("split() ok and end...");
+ return readerSplitConfigs;
+ }
+
+ }
+
+ public static class Task extends Reader.Task {
+ private static final Logger LOG = LoggerFactory.getLogger(Task.class);
+
+ private Configuration taskConfig;
+
+ private String accessId;
+ private String accessKey;
+ private String endpoint;
+ private String project;
+ private String topic;
+ private String shardId;
+ private Long beginTimestampMillis;
+ private Long endTimestampMillis;
+ private int batchSize;
+ private List columns;
+ private RecordSchema schema;
+ private String timeStampUnit;
+
+ DatahubClient datahubClient;
+
+ @Override
+ public void init() {
+ this.taskConfig = super.getPluginJobConf();
+
+ this.accessId = this.taskConfig.getString(Key.ACCESSKEYID);
+ this.accessKey = this.taskConfig.getString(Key.ACCESSKEYSECRET);
+ this.endpoint = this.taskConfig.getString(Key.ENDPOINT);
+ this.project = this.taskConfig.getString(Key.PROJECT);
+ this.topic = this.taskConfig.getString(Key.TOPIC);
+ this.shardId = this.taskConfig.getString(Key.SHARDID);
+ this.batchSize = this.taskConfig.getInt(Key.BATCHSIZE, 1024);
+ this.timeStampUnit = this.taskConfig.getString(Key.TIMESTAMP_UNIT, "MICROSECOND");
+ try {
+ this.beginTimestampMillis = DatahubReaderUtils.getUnixTimeFromDateTime(this.taskConfig.getString(Key.BEGINDATETIME));
+ } catch (ParseException e) {
+ }
+
+ try {
+ this.endTimestampMillis = DatahubReaderUtils.getUnixTimeFromDateTime(this.taskConfig.getString(Key.ENDDATETIME));
+ } catch (ParseException e) {
+ }
+
+ this.columns = this.taskConfig.getList(Key.COLUMN, String.class);
+
+ this.datahubClient = DatahubClientHelper.getDatahubClient(this.taskConfig);
+
+
+ this.schema = DatahubReaderUtils.getDatahubSchemaWithRetry(this.datahubClient, this.project, topic);
+
+ LOG.info("init datahub reader task finished.project:{} topic:{} batchSize:{}", project, topic, batchSize);
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+ @Override
+ public void startRead(RecordSender recordSender) {
+ LOG.info("read start");
+
+ String beginCursor = DatahubReaderUtils.getCursorWithRetry(this.datahubClient, this.project,
+ this.topic, this.shardId, this.beginTimestampMillis);
+ String endCursor = DatahubReaderUtils.getCursorWithRetry(this.datahubClient, this.project,
+ this.topic, this.shardId, this.endTimestampMillis);
+
+ if (beginCursor == null) {
+ LOG.info("Shard:{} has no data!", this.shardId);
+ return;
+ } else if (endCursor == null) {
+ endCursor = DatahubReaderUtils.getLatestCursorWithRetry(this.datahubClient, this.project,
+ this.topic, this.shardId);
+ }
+
+ String curCursor = beginCursor;
+
+ boolean exit = false;
+
+ while (true) {
+
+ GetRecordsResult result = DatahubReaderUtils.getRecordsResultWithRetry(this.datahubClient, this.project, this.topic,
+ this.shardId, this.batchSize, curCursor, this.schema);
+
+ List records = result.getRecords();
+ if (records.size() > 0) {
+ for (RecordEntry record : records) {
+ if (record.getSystemTime() >= this.endTimestampMillis) {
+ exit = true;
+ break;
+ }
+
+ HashMap dataMap = new HashMap();
+ List fields = ((TupleRecordData) record.getRecordData()).getRecordSchema().getFields();
+ for (int i = 0; i < fields.size(); i++) {
+ Field field = fields.get(i);
+ Column column = DatahubReaderUtils.getColumnFromField(record, field, this.timeStampUnit);
+ dataMap.put(field.getName(), column);
+ }
+
+ Record dataxRecord = recordSender.createRecord();
+
+ if (null != this.columns && 1 == this.columns.size()) {
+ String columnsInStr = columns.get(0).toString();
+ if ("\"*\"".equals(columnsInStr) || "*".equals(columnsInStr)) {
+ for (int i = 0; i < fields.size(); i++) {
+ dataxRecord.addColumn(dataMap.get(fields.get(i).getName()));
+ }
+
+ } else {
+ if (dataMap.containsKey(columnsInStr)) {
+ dataxRecord.addColumn(dataMap.get(columnsInStr));
+ } else {
+ dataxRecord.addColumn(new StringColumn(null));
+ }
+ }
+ } else {
+ for (String col : this.columns) {
+ if (dataMap.containsKey(col)) {
+ dataxRecord.addColumn(dataMap.get(col));
+ } else {
+ dataxRecord.addColumn(new StringColumn(null));
+ }
+ }
+ }
+
+ recordSender.sendToWriter(dataxRecord);
+ }
+ } else {
+ break;
+ }
+
+ if (exit) {
+ break;
+ }
+
+ curCursor = result.getNextCursor();
+ }
+
+
+ LOG.info("end read datahub shard...");
+ }
+
+ }
+
+}
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReaderErrorCode.java b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReaderErrorCode.java
new file mode 100644
index 00000000..949a66f0
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReaderErrorCode.java
@@ -0,0 +1,35 @@
+package com.alibaba.datax.plugin.reader.datahubreader;
+
+import com.alibaba.datax.common.spi.ErrorCode;
+
+public enum DatahubReaderErrorCode implements ErrorCode {
+ BAD_CONFIG_VALUE("DatahubReader-00", "The value you configured is invalid."),
+ LOG_HUB_ERROR("DatahubReader-01","Datahub exception"),
+ REQUIRE_VALUE("DatahubReader-02","Missing parameters"),
+ EMPTY_LOGSTORE_VALUE("DatahubReader-03","There is no shard under this LogStore");
+
+
+ private final String code;
+ private final String description;
+
+ private DatahubReaderErrorCode(String code, String description) {
+ this.code = code;
+ this.description = description;
+ }
+
+ @Override
+ public String getCode() {
+ return this.code;
+ }
+
+ @Override
+ public String getDescription() {
+ return this.description;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Code:[%s], Description:[%s]. ", this.code,
+ this.description);
+ }
+}
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReaderUtils.java b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReaderUtils.java
new file mode 100644
index 00000000..6c3455df
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReaderUtils.java
@@ -0,0 +1,200 @@
+package com.alibaba.datax.plugin.reader.datahubreader;
+
+import java.math.BigDecimal;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+import com.alibaba.datax.common.element.*;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.DataXCaseEnvUtil;
+import com.alibaba.datax.common.util.RetryUtil;
+
+import com.aliyun.datahub.client.DatahubClient;
+import com.aliyun.datahub.client.exception.InvalidParameterException;
+import com.aliyun.datahub.client.model.*;
+
+public class DatahubReaderUtils {
+
+ public static long getUnixTimeFromDateTime(String dateTime) throws ParseException {
+ try {
+ String format = Constant.DATETIME_FORMAT;
+ SimpleDateFormat simpleDateFormat = new SimpleDateFormat(format);
+ return simpleDateFormat.parse(dateTime).getTime();
+ } catch (ParseException ignored) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "Invalid DateTime[" + dateTime + "]!");
+ }
+ }
+
+ public static List getShardsWithRetry(final DatahubClient datahubClient, final String project, final String topic) {
+
+ List shards = null;
+ try {
+ shards = RetryUtil.executeWithRetry(new Callable>() {
+ @Override
+ public List call() throws Exception {
+ ListShardResult listShardResult = datahubClient.listShard(project, topic);
+ return listShardResult.getShards();
+ }
+ }, DataXCaseEnvUtil.getRetryTimes(7), DataXCaseEnvUtil.getRetryInterval(1000L), DataXCaseEnvUtil.getRetryExponential(true));
+
+ } catch (Exception e) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "get Shards error, please check ! detail error messsage: " + e.toString());
+ }
+ return shards;
+ }
+
+ public static String getCursorWithRetry(final DatahubClient datahubClient, final String project, final String topic,
+ final String shardId, final long timestamp) {
+
+ String cursor;
+ try {
+ cursor = RetryUtil.executeWithRetry(new Callable() {
+ @Override
+ public String call() throws Exception {
+ try {
+ return datahubClient.getCursor(project, topic, shardId, CursorType.SYSTEM_TIME, timestamp).getCursor();
+ } catch (InvalidParameterException e) {
+ if (e.getErrorMessage().indexOf("Time in seek request is out of range") >= 0) {
+ return null;
+ } else {
+ throw e;
+ }
+
+ }
+ }
+ }, DataXCaseEnvUtil.getRetryTimes(7), DataXCaseEnvUtil.getRetryInterval(1000L), DataXCaseEnvUtil.getRetryExponential(true));
+
+ } catch (Exception e) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "get Cursor error, please check ! detail error messsage: " + e.toString());
+ }
+ return cursor;
+ }
+
+ public static String getLatestCursorWithRetry(final DatahubClient datahubClient, final String project, final String topic,
+ final String shardId) {
+
+ String cursor;
+ try {
+ cursor = RetryUtil.executeWithRetry(new Callable() {
+ @Override
+ public String call() throws Exception {
+ return datahubClient.getCursor(project, topic, shardId, CursorType.LATEST).getCursor();
+ }
+ }, DataXCaseEnvUtil.getRetryTimes(7), DataXCaseEnvUtil.getRetryInterval(1000L), DataXCaseEnvUtil.getRetryExponential(true));
+
+ } catch (Exception e) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "get Cursor error, please check ! detail error messsage: " + e.toString());
+ }
+ return cursor;
+ }
+
+ public static RecordSchema getDatahubSchemaWithRetry(final DatahubClient datahubClient, final String project, final String topic) {
+
+ RecordSchema schema;
+ try {
+ schema = RetryUtil.executeWithRetry(new Callable() {
+ @Override
+ public RecordSchema call() throws Exception {
+ return datahubClient.getTopic(project, topic).getRecordSchema();
+ }
+ }, DataXCaseEnvUtil.getRetryTimes(7), DataXCaseEnvUtil.getRetryInterval(1000L), DataXCaseEnvUtil.getRetryExponential(true));
+
+ } catch (Exception e) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "get Topic Schema error, please check ! detail error messsage: " + e.toString());
+ }
+ return schema;
+ }
+
+ public static GetRecordsResult getRecordsResultWithRetry(final DatahubClient datahubClient, final String project,
+ final String topic, final String shardId, final int batchSize, final String cursor, final RecordSchema schema) {
+
+ GetRecordsResult result;
+ try {
+ result = RetryUtil.executeWithRetry(new Callable() {
+ @Override
+ public GetRecordsResult call() throws Exception {
+ return datahubClient.getRecords(project, topic, shardId, schema, cursor, batchSize);
+ }
+ }, DataXCaseEnvUtil.getRetryTimes(7), DataXCaseEnvUtil.getRetryInterval(1000L), DataXCaseEnvUtil.getRetryExponential(true));
+
+ } catch (Exception e) {
+ throw DataXException.asDataXException(DatahubReaderErrorCode.BAD_CONFIG_VALUE,
+ "get Record Result error, please check ! detail error messsage: " + e.toString());
+ }
+ return result;
+
+ }
+
+ public static Column getColumnFromField(RecordEntry record, Field field, String timeStampUnit) {
+ Column col = null;
+ TupleRecordData o = (TupleRecordData) record.getRecordData();
+
+ switch (field.getType()) {
+ case SMALLINT:
+ Short shortValue = ((Short) o.getField(field.getName()));
+ col = new LongColumn(shortValue == null ? null: shortValue.longValue());
+ break;
+ case INTEGER:
+ col = new LongColumn((Integer) o.getField(field.getName()));
+ break;
+ case BIGINT: {
+ col = new LongColumn((Long) o.getField(field.getName()));
+ break;
+ }
+ case TINYINT: {
+ Byte byteValue = ((Byte) o.getField(field.getName()));
+ col = new LongColumn(byteValue == null ? null : byteValue.longValue());
+ break;
+ }
+ case BOOLEAN: {
+ col = new BoolColumn((Boolean) o.getField(field.getName()));
+ break;
+ }
+ case FLOAT:
+ col = new DoubleColumn((Float) o.getField(field.getName()));
+ break;
+ case DOUBLE: {
+ col = new DoubleColumn((Double) o.getField(field.getName()));
+ break;
+ }
+ case STRING: {
+ col = new StringColumn((String) o.getField(field.getName()));
+ break;
+ }
+ case DECIMAL: {
+ BigDecimal value = (BigDecimal) o.getField(field.getName());
+ col = new DoubleColumn(value == null ? null : value.doubleValue());
+ break;
+ }
+ case TIMESTAMP: {
+ Long value = (Long) o.getField(field.getName());
+
+ if ("MILLISECOND".equals(timeStampUnit)) {
+ // MILLISECOND, 13位精度，直接 new Date()
+ col = new DateColumn(value == null ? null : new Date(value));
+ }
+ else if ("SECOND".equals(timeStampUnit)){
+ col = new DateColumn(value == null ? null : new Date(value * 1000));
+ }
+ else {
+ // 默认都是 MICROSECOND, 16位精度， 和之前的逻辑保持一致。
+ col = new DateColumn(value == null ? null : new Date(value / 1000));
+ }
+ break;
+ }
+ default:
+ throw new RuntimeException("Unknown column type: " + field.getType());
+ }
+
+ return col;
+ }
+
+}
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubWriterErrorCode.java b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubWriterErrorCode.java
new file mode 100644
index 00000000..c8633ea8
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubWriterErrorCode.java
@@ -0,0 +1,37 @@
+package com.alibaba.datax.plugin.reader.datahubreader;
+
+import com.alibaba.datax.common.spi.ErrorCode;
+import com.alibaba.datax.common.util.MessageSource;
+
+public enum DatahubWriterErrorCode implements ErrorCode {
+ MISSING_REQUIRED_VALUE("DatahubWriter-01", MessageSource.loadResourceBundle(DatahubWriterErrorCode.class).message("errorcode.missing_required_value")),
+ INVALID_CONFIG_VALUE("DatahubWriter-02", MessageSource.loadResourceBundle(DatahubWriterErrorCode.class).message("errorcode.invalid_config_value")),
+ GET_TOPOIC_INFO_FAIL("DatahubWriter-03", MessageSource.loadResourceBundle(DatahubWriterErrorCode.class).message("errorcode.get_topic_info_fail")),
+ WRITE_DATAHUB_FAIL("DatahubWriter-04", MessageSource.loadResourceBundle(DatahubWriterErrorCode.class).message("errorcode.write_datahub_fail")),
+ SCHEMA_NOT_MATCH("DatahubWriter-05", MessageSource.loadResourceBundle(DatahubWriterErrorCode.class).message("errorcode.schema_not_match")),
+ ;
+
+ private final String code;
+ private final String description;
+
+ private DatahubWriterErrorCode(String code, String description) {
+ this.code = code;
+ this.description = description;
+ }
+
+ @Override
+ public String getCode() {
+ return this.code;
+ }
+
+ @Override
+ public String getDescription() {
+ return this.description;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Code:[%s], Description:[%s]. ", this.code,
+ this.description);
+ }
+}
\ No newline at end of file
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Key.java b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Key.java
new file mode 100644
index 00000000..3cb84b4b
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Key.java
@@ -0,0 +1,35 @@
+package com.alibaba.datax.plugin.reader.datahubreader;
+
+public final class Key {
+
+ /**
+ * 此处声明插件用到的需要插件使用者提供的配置项
+ */
+ public static final String ENDPOINT = "endpoint";
+
+ public static final String ACCESSKEYID = "accessId";
+
+ public static final String ACCESSKEYSECRET = "accessKey";
+
+ public static final String PROJECT = "project";
+
+ public static final String TOPIC = "topic";
+
+ public static final String BEGINDATETIME = "beginDateTime";
+
+ public static final String ENDDATETIME = "endDateTime";
+
+ public static final String BATCHSIZE = "batchSize";
+
+ public static final String COLUMN = "column";
+
+ public static final String SHARDID = "shardId";
+
+ public static final String CONFIG_KEY_ENDPOINT = "endpoint";
+ public static final String CONFIG_KEY_ACCESS_ID = "accessId";
+ public static final String CONFIG_KEY_ACCESS_KEY = "accessKey";
+
+
+ public static final String TIMESTAMP_UNIT = "timeStampUnit";
+
+}
\ No newline at end of file
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings.properties b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings.properties
new file mode 100644
index 00000000..e85c8ab3
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings.properties
@@ -0,0 +1,5 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_en_US.properties b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_en_US.properties
new file mode 100644
index 00000000..31a291e6
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_en_US.properties
@@ -0,0 +1,5 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.
\ No newline at end of file
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_ja_JP.properties b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_ja_JP.properties
new file mode 100644
index 00000000..31a291e6
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_ja_JP.properties
@@ -0,0 +1,5 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.
\ No newline at end of file
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_zh_CN.properties b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_zh_CN.properties
new file mode 100644
index 00000000..31a291e6
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_zh_CN.properties
@@ -0,0 +1,5 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.
\ No newline at end of file
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_zh_HK.properties b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_zh_HK.properties
new file mode 100644
index 00000000..c6a3a0e0
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_zh_HK.properties
@@ -0,0 +1,9 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.errorcode.missing_required_value=您缺失了必須填寫的參數值.
+errorcode.invalid_config_value=您的參數配寘錯誤.
+errorcode.get_topic_info_fail=獲取shard清單失敗.
+errorcode.write_datahub_fail=寫數據失敗.
+errorcode.schema_not_match=數據格式錯誤.
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_zh_TW.properties b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_zh_TW.properties
new file mode 100644
index 00000000..c6a3a0e0
--- /dev/null
+++ b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/LocalStrings_zh_TW.properties
@@ -0,0 +1,9 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.errorcode.missing_required_value=您缺失了必須填寫的參數值.
+errorcode.invalid_config_value=您的參數配寘錯誤.
+errorcode.get_topic_info_fail=獲取shard清單失敗.
+errorcode.write_datahub_fail=寫數據失敗.
+errorcode.schema_not_match=數據格式錯誤.
diff --git a/datahubreader/src/main/resources/job_config_template.json b/datahubreader/src/main/resources/job_config_template.json
new file mode 100644
index 00000000..eaf89804
--- /dev/null
+++ b/datahubreader/src/main/resources/job_config_template.json
@@ -0,0 +1,14 @@
+{
+ "name": "datahubreader",
+ "parameter": {
+ "endpoint":"",
+ "accessId": "",
+ "accessKey": "",
+ "project": "",
+ "topic": "",
+ "beginDateTime": "20180913121019",
+ "endDateTime": "20180913121119",
+ "batchSize": 1024,
+ "column": []
+ }
+}
\ No newline at end of file
diff --git a/datahubreader/src/main/resources/plugin.json b/datahubreader/src/main/resources/plugin.json
new file mode 100644
index 00000000..47b1c86b
--- /dev/null
+++ b/datahubreader/src/main/resources/plugin.json
@@ -0,0 +1,6 @@
+{
+ "name": "datahubreader",
+ "class": "com.alibaba.datax.plugin.reader.datahubreader.DatahubReader",
+ "description": "datahub reader",
+ "developer": "alibaba"
+}
\ No newline at end of file
diff --git a/datahubwriter/pom.xml b/datahubwriter/pom.xml
new file mode 100644
index 00000000..1ee1fe9b
--- /dev/null
+++ b/datahubwriter/pom.xml
@@ -0,0 +1,79 @@
+
+
+
+ datax-all
+ com.alibaba.datax
+ 0.0.1-SNAPSHOT
+
+ 4.0.0
+
+ datahubwriter
+
+ 0.0.1-SNAPSHOT
+
+
+
+ com.alibaba.datax
+ datax-common
+ ${datax-project-version}
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+ ch.qos.logback
+ logback-classic
+
+
+ com.aliyun.datahub
+ aliyun-sdk-datahub
+ 2.21.6-public
+
+
+ junit
+ junit
+ 4.12
+ test
+
+
+
+
+
+
+
+ maven-compiler-plugin
+
+ ${jdk-version}
+ ${jdk-version}
+ ${project-sourceEncoding}
+
+
+
+
+ maven-assembly-plugin
+
+
+ src/main/assembly/package.xml
+
+ datax
+
+
+
+ dwzip
+ package
+
+ single
+
+
+
+
+
+
+
diff --git a/datahubwriter/src/main/assembly/package.xml b/datahubwriter/src/main/assembly/package.xml
new file mode 100644
index 00000000..aaef9f99
--- /dev/null
+++ b/datahubwriter/src/main/assembly/package.xml
@@ -0,0 +1,34 @@
+
+
+
+ dir
+
+ false
+
+
+ src/main/resources
+
+ plugin.json
+
+ plugin/writer/datahubwriter
+
+
+ target/
+
+ datahubwriter-0.0.1-SNAPSHOT.jar
+
+ plugin/writer/datahubwriter
+
+
+
+
+
+ false
+ plugin/writer/datahubwriter/libs
+ runtime
+
+
+
diff --git a/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubClientHelper.java b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubClientHelper.java
new file mode 100644
index 00000000..c25d1210
--- /dev/null
+++ b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubClientHelper.java
@@ -0,0 +1,43 @@
+package com.alibaba.datax.plugin.writer.datahubwriter;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.TypeReference;
+import com.aliyun.datahub.client.DatahubClient;
+import com.aliyun.datahub.client.DatahubClientBuilder;
+import com.aliyun.datahub.client.auth.Account;
+import com.aliyun.datahub.client.auth.AliyunAccount;
+import com.aliyun.datahub.client.common.DatahubConfig;
+import com.aliyun.datahub.client.http.HttpConfig;
+
+public class DatahubClientHelper {
+ public static DatahubClient getDatahubClient(Configuration jobConfig) {
+ String accessId = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ACCESS_ID,
+ DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ String accessKey = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ACCESS_KEY,
+ DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ String endpoint = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ENDPOINT,
+ DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ Account account = new AliyunAccount(accessId, accessKey);
+ // 是否开启二进制传输，服务端2.12版本开始支持
+ boolean enableBinary = jobConfig.getBool("enableBinary", false);
+ DatahubConfig datahubConfig = new DatahubConfig(endpoint, account, enableBinary);
+ // HttpConfig可不设置，不设置时采用默认值
+ // 读写数据推荐打开网络传输 LZ4压缩
+ HttpConfig httpConfig = null;
+ String httpConfigStr = jobConfig.getString("httpConfig");
+ if (StringUtils.isNotBlank(httpConfigStr)) {
+ httpConfig = JSON.parseObject(httpConfigStr, new TypeReference() {
+ });
+ }
+
+ DatahubClientBuilder builder = DatahubClientBuilder.newBuilder().setDatahubConfig(datahubConfig);
+ if (null != httpConfig) {
+ builder.setHttpConfig(httpConfig);
+ }
+ DatahubClient datahubClient = builder.build();
+ return datahubClient;
+ }
+}
diff --git a/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubWriter.java b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubWriter.java
new file mode 100644
index 00000000..cd414fc5
--- /dev/null
+++ b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubWriter.java
@@ -0,0 +1,355 @@
+package com.alibaba.datax.plugin.writer.datahubwriter;
+
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+import com.alibaba.datax.common.spi.Writer;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.util.DataXCaseEnvUtil;
+import com.alibaba.datax.common.util.RetryUtil;
+import com.alibaba.fastjson2.JSON;
+import com.aliyun.datahub.client.DatahubClient;
+import com.aliyun.datahub.client.model.FieldType;
+import com.aliyun.datahub.client.model.GetTopicResult;
+import com.aliyun.datahub.client.model.ListShardResult;
+import com.aliyun.datahub.client.model.PutErrorEntry;
+import com.aliyun.datahub.client.model.PutRecordsResult;
+import com.aliyun.datahub.client.model.RecordEntry;
+import com.aliyun.datahub.client.model.RecordSchema;
+import com.aliyun.datahub.client.model.RecordType;
+import com.aliyun.datahub.client.model.ShardEntry;
+import com.aliyun.datahub.client.model.ShardState;
+import com.aliyun.datahub.client.model.TupleRecordData;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+
+public class DatahubWriter extends Writer {
+
+ /**
+ * Job 中的方法仅执行一次，Task 中方法会由框架启动多个 Task 线程并行执行。
+ *
+ * 整个 Writer 执行流程是：
+ *
+ * Job类init-->prepare-->split
+ *
+ * Task类init-->prepare-->startWrite-->post-->destroy
+ * Task类init-->prepare-->startWrite-->post-->destroy
+ *
+ * Job类post-->destroy
+ *
+ */
+ public static class Job extends Writer.Job {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(Job.class);
+
+ private Configuration jobConfig = null;
+
+ @Override
+ public void init() {
+ this.jobConfig = super.getPluginJobConf();
+ jobConfig.getNecessaryValue(Key.CONFIG_KEY_ENDPOINT, DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ jobConfig.getNecessaryValue(Key.CONFIG_KEY_ACCESS_ID, DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ jobConfig.getNecessaryValue(Key.CONFIG_KEY_ACCESS_KEY, DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ jobConfig.getNecessaryValue(Key.CONFIG_KEY_PROJECT, DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ jobConfig.getNecessaryValue(Key.CONFIG_KEY_TOPIC, DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ }
+
+ @Override
+ public void prepare() {
+ String project = jobConfig.getNecessaryValue(Key.CONFIG_KEY_PROJECT,
+ DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ String topic = jobConfig.getNecessaryValue(Key.CONFIG_KEY_TOPIC,
+ DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ RecordType recordType = null;
+ DatahubClient client = DatahubClientHelper.getDatahubClient(this.jobConfig);
+ try {
+ GetTopicResult getTopicResult = client.getTopic(project, topic);
+ recordType = getTopicResult.getRecordType();
+ } catch (Exception e) {
+ LOG.warn("get topic type error: {}", e.getMessage());
+ }
+ if (null != recordType) {
+ if (recordType == RecordType.BLOB) {
+ throw DataXException.asDataXException(DatahubWriterErrorCode.WRITE_DATAHUB_FAIL,
+ "DatahubWriter only support 'Tuple' RecordType now, but your RecordType is 'BLOB'");
+ }
+ }
+ }
+
+ @Override
+ public List split(int mandatoryNumber) {
+ List configs = new ArrayList();
+ for (int i = 0; i < mandatoryNumber; ++i) {
+ configs.add(jobConfig.clone());
+ }
+ return configs;
+ }
+
+ @Override
+ public void post() {}
+
+ @Override
+ public void destroy() {}
+
+ }
+
+ public static class Task extends Writer.Task {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(Task.class);
+ private static final List FATAL_ERRORS_DEFAULT = Arrays.asList(
+ "InvalidParameterM",
+ "MalformedRecord",
+ "INVALID_SHARDID",
+ "NoSuchTopic",
+ "NoSuchShard"
+ );
+
+ private Configuration taskConfig;
+ private DatahubClient client;
+ private String project;
+ private String topic;
+ private List shards;
+ private int maxCommitSize;
+ private int maxRetryCount;
+ private RecordSchema schema;
+ private long retryInterval;
+ private Random random;
+ private List column;
+ private List columnIndex;
+ private boolean enableColumnConfig;
+ private List fatalErrors;
+
+ @Override
+ public void init() {
+ this.taskConfig = super.getPluginJobConf();
+ project = taskConfig.getNecessaryValue(Key.CONFIG_KEY_PROJECT, DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ topic = taskConfig.getNecessaryValue(Key.CONFIG_KEY_TOPIC, DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
+ maxCommitSize = taskConfig.getInt(Key.CONFIG_KEY_MAX_COMMIT_SIZE, 1024*1024);
+ maxRetryCount = taskConfig.getInt(Key.CONFIG_KEY_MAX_RETRY_COUNT, 500);
+ this.retryInterval = taskConfig.getInt(Key.RETRY_INTERVAL, 650);
+ this.random = new Random();
+ this.column = this.taskConfig.getList(Key.CONFIG_KEY_COLUMN, String.class);
+ // ["*"]
+ if (null != this.column && 1 == this.column.size()) {
+ if (StringUtils.equals("*", this.column.get(0))) {
+ this.column = null;
+ }
+ }
+ this.columnIndex = new ArrayList();
+ // 留个开关保平安
+ this.enableColumnConfig = this.taskConfig.getBool("enableColumnConfig", true);
+ this.fatalErrors = this.taskConfig.getList("fatalErrors", Task.FATAL_ERRORS_DEFAULT, String.class);
+ this.client = DatahubClientHelper.getDatahubClient(this.taskConfig);
+ }
+
+ @Override
+ public void prepare() {
+ final String shardIdConfig = this.taskConfig.getString(Key.CONFIG_KEY_SHARD_ID);
+ this.shards = new ArrayList();
+ try {
+ RetryUtil.executeWithRetry(new Callable() {
+ @Override
+ public Void call() throws Exception {
+ ListShardResult result = client.listShard(project, topic);
+ if (StringUtils.isNotBlank(shardIdConfig)) {
+ shards.add(shardIdConfig);
+ } else {
+ for (ShardEntry shard : result.getShards()) {
+ if (shard.getState() == ShardState.ACTIVE || shard.getState() == ShardState.OPENING) {
+ shards.add(shard.getShardId());
+ }
+ }
+ }
+ schema = client.getTopic(project, topic).getRecordSchema();
+ return null;
+ }
+ }, DataXCaseEnvUtil.getRetryTimes(5), DataXCaseEnvUtil.getRetryInterval(10000L), DataXCaseEnvUtil.getRetryExponential(false));
+ } catch (Exception e) {
+ throw DataXException.asDataXException(DatahubWriterErrorCode.GET_TOPOIC_INFO_FAIL,
+ "get topic info failed", e);
+ }
+ LOG.info("datahub topic {} shard to write: {}", this.topic, JSON.toJSONString(this.shards));
+ LOG.info("datahub topic {} has schema: {}", this.topic, JSON.toJSONString(this.schema));
+
+ // 根据 schmea 顺序 和用户配置的 column，计算写datahub的顺序关系，以支持列换序
+ // 后续统一使用 columnIndex 的顺位关系写 datahub
+ int totalSize = this.schema.getFields().size();
+ if (null != this.column && !this.column.isEmpty() && this.enableColumnConfig) {
+ for (String eachCol : this.column) {
+ int indexFound = -1;
+ for (int i = 0; i < totalSize; i++) {
+ // warn: 大小写ignore
+ if (StringUtils.equalsIgnoreCase(eachCol, this.schema.getField(i).getName())) {
+ indexFound = i;
+ break;
+ }
+ }
+ if (indexFound >= 0) {
+ this.columnIndex.add(indexFound);
+ } else {
+ throw DataXException.asDataXException(DatahubWriterErrorCode.SCHEMA_NOT_MATCH,
+ String.format("can not find column %s in datahub topic %s", eachCol, this.topic));
+ }
+ }
+ } else {
+ for (int i = 0; i < totalSize; i++) {
+ this.columnIndex.add(i);
+ }
+ }
+ }
+
+ @Override
+ public void startWrite(RecordReceiver recordReceiver) {
+ Record record;
+ List records = new ArrayList();
+ String shardId = null;
+ if (1 == this.shards.size()) {
+ shardId = shards.get(0);
+ } else {
+ shardId = shards.get(this.random.nextInt(shards.size()));
+ }
+ int commitSize = 0;
+ try {
+ while ((record = recordReceiver.getFromReader()) != null) {
+ RecordEntry dhRecord = convertRecord(record, shardId);
+ if (dhRecord != null) {
+ records.add(dhRecord);
+ }
+ commitSize += record.getByteSize();
+ if (commitSize >= maxCommitSize) {
+ commit(records);
+ records.clear();
+ commitSize = 0;
+ if (1 == this.shards.size()) {
+ shardId = shards.get(0);
+ } else {
+ shardId = shards.get(this.random.nextInt(shards.size()));
+ }
+ }
+ }
+ if (commitSize > 0) {
+ commit(records);
+ }
+ } catch (Exception e) {
+ throw DataXException.asDataXException(
+ DatahubWriterErrorCode.WRITE_DATAHUB_FAIL, e);
+ }
+ }
+
+ @Override
+ public void post() {}
+
+ @Override
+ public void destroy() {}
+
+ private void commit(List records) throws InterruptedException {
+ PutRecordsResult result = client.putRecords(project, topic, records);
+ if (result.getFailedRecordCount() > 0) {
+ for (int i = 0; i < maxRetryCount; ++i) {
+ boolean limitExceededMessagePrinted = false;
+ for (PutErrorEntry error : result.getPutErrorEntries()) {
+ // 如果是 LimitExceeded 这样打印日志，不能每行记录打印一次了
+ if (StringUtils.equalsIgnoreCase("LimitExceeded", error.getErrorcode())) {
+ if (!limitExceededMessagePrinted) {
+ LOG.warn("write record error, request id: {}, error code: {}, error message: {}",
+ result.getRequestId(), error.getErrorcode(), error.getMessage());
+ limitExceededMessagePrinted = true;
+ }
+ } else {
+ LOG.error("write record error, request id: {}, error code: {}, error message: {}",
+ result.getRequestId(), error.getErrorcode(), error.getMessage());
+ }
+ if (this.fatalErrors.contains(error.getErrorcode())) {
+ throw DataXException.asDataXException(
+ DatahubWriterErrorCode.WRITE_DATAHUB_FAIL,
+ error.getMessage());
+ }
+ }
+
+ if (this.retryInterval >= 0) {
+ Thread.sleep(this.retryInterval);
+ } else {
+ Thread.sleep(new Random().nextInt(700) + 300);
+ }
+
+ result = client.putRecords(project, topic, result.getFailedRecords());
+ if (result.getFailedRecordCount() == 0) {
+ return;
+ }
+ }
+ throw DataXException.asDataXException(
+ DatahubWriterErrorCode.WRITE_DATAHUB_FAIL,
+ "write datahub failed");
+ }
+ }
+
+ private RecordEntry convertRecord(Record dxRecord, String shardId) {
+ try {
+ RecordEntry dhRecord = new RecordEntry();
+ dhRecord.setShardId(shardId);
+ TupleRecordData data = new TupleRecordData(this.schema);
+ for (int i = 0; i < this.columnIndex.size(); ++i) {
+ int orderInSchema = this.columnIndex.get(i);
+ FieldType type = this.schema.getField(orderInSchema).getType();
+ Column column = dxRecord.getColumn(i);
+ switch (type) {
+ case BIGINT:
+ data.setField(orderInSchema, column.asLong());
+ break;
+ case DOUBLE:
+ data.setField(orderInSchema, column.asDouble());
+ break;
+ case STRING:
+ data.setField(orderInSchema, column.asString());
+ break;
+ case BOOLEAN:
+ data.setField(orderInSchema, column.asBoolean());
+ break;
+ case TIMESTAMP:
+ if (null == column.asDate()) {
+ data.setField(orderInSchema, null);
+ } else {
+ data.setField(orderInSchema, column.asDate().getTime() * 1000);
+ }
+ break;
+ case DECIMAL:
+ // warn
+ data.setField(orderInSchema, column.asBigDecimal());
+ break;
+ case INTEGER:
+ data.setField(orderInSchema, column.asLong());
+ break;
+ case FLOAT:
+ data.setField(orderInSchema, column.asDouble());
+ break;
+ case TINYINT:
+ data.setField(orderInSchema, column.asLong());
+ break;
+ case SMALLINT:
+ data.setField(orderInSchema, column.asLong());
+ break;
+ default:
+ throw DataXException.asDataXException(
+ DatahubWriterErrorCode.SCHEMA_NOT_MATCH,
+ String.format("does not support type: %s", type));
+ }
+ }
+ dhRecord.setRecordData(data);
+ return dhRecord;
+ } catch (Exception e) {
+ super.getTaskPluginCollector().collectDirtyRecord(dxRecord, e, "convert recor failed");
+ }
+ return null;
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubWriterErrorCode.java b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubWriterErrorCode.java
new file mode 100644
index 00000000..ad03abd1
--- /dev/null
+++ b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubWriterErrorCode.java
@@ -0,0 +1,37 @@
+package com.alibaba.datax.plugin.writer.datahubwriter;
+
+import com.alibaba.datax.common.spi.ErrorCode;
+import com.alibaba.datax.common.util.MessageSource;
+
+public enum DatahubWriterErrorCode implements ErrorCode {
+ MISSING_REQUIRED_VALUE("DatahubWriter-01", MessageSource.loadResourceBundle(DatahubWriterErrorCode.class).message("errorcode.missing_required_value")),
+ INVALID_CONFIG_VALUE("DatahubWriter-02", MessageSource.loadResourceBundle(DatahubWriterErrorCode.class).message("errorcode.invalid_config_value")),
+ GET_TOPOIC_INFO_FAIL("DatahubWriter-03", MessageSource.loadResourceBundle(DatahubWriterErrorCode.class).message("errorcode.get_topic_info_fail")),
+ WRITE_DATAHUB_FAIL("DatahubWriter-04", MessageSource.loadResourceBundle(DatahubWriterErrorCode.class).message("errorcode.write_datahub_fail")),
+ SCHEMA_NOT_MATCH("DatahubWriter-05", MessageSource.loadResourceBundle(DatahubWriterErrorCode.class).message("errorcode.schema_not_match")),
+ ;
+
+ private final String code;
+ private final String description;
+
+ private DatahubWriterErrorCode(String code, String description) {
+ this.code = code;
+ this.description = description;
+ }
+
+ @Override
+ public String getCode() {
+ return this.code;
+ }
+
+ @Override
+ public String getDescription() {
+ return this.description;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Code:[%s], Description:[%s]. ", this.code,
+ this.description);
+ }
+}
\ No newline at end of file
diff --git a/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/Key.java b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/Key.java
new file mode 100644
index 00000000..5f179234
--- /dev/null
+++ b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/Key.java
@@ -0,0 +1,26 @@
+package com.alibaba.datax.plugin.writer.datahubwriter;
+
+public final class Key {
+
+ /**
+ * 此处声明插件用到的需要插件使用者提供的配置项
+ */
+ public static final String CONFIG_KEY_ENDPOINT = "endpoint";
+ public static final String CONFIG_KEY_ACCESS_ID = "accessId";
+ public static final String CONFIG_KEY_ACCESS_KEY = "accessKey";
+ public static final String CONFIG_KEY_PROJECT = "project";
+ public static final String CONFIG_KEY_TOPIC = "topic";
+ public static final String CONFIG_KEY_WRITE_MODE = "mode";
+ public static final String CONFIG_KEY_SHARD_ID = "shardId";
+ public static final String CONFIG_KEY_MAX_COMMIT_SIZE = "maxCommitSize";
+ public static final String CONFIG_KEY_MAX_RETRY_COUNT = "maxRetryCount";
+
+ public static final String CONFIG_VALUE_SEQUENCE_MODE = "sequence";
+ public static final String CONFIG_VALUE_RANDOM_MODE = "random";
+
+ public final static String MAX_RETRY_TIME = "maxRetryTime";
+
+ public final static String RETRY_INTERVAL = "retryInterval";
+
+ public final static String CONFIG_KEY_COLUMN = "column";
+}
diff --git a/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings.properties b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings.properties
new file mode 100644
index 00000000..e85c8ab3
--- /dev/null
+++ b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings.properties
@@ -0,0 +1,5 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.
diff --git a/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_en_US.properties b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_en_US.properties
new file mode 100644
index 00000000..31a291e6
--- /dev/null
+++ b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_en_US.properties
@@ -0,0 +1,5 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.
\ No newline at end of file
diff --git a/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_ja_JP.properties b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_ja_JP.properties
new file mode 100644
index 00000000..31a291e6
--- /dev/null
+++ b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_ja_JP.properties
@@ -0,0 +1,5 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.
\ No newline at end of file
diff --git a/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_zh_CN.properties b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_zh_CN.properties
new file mode 100644
index 00000000..31a291e6
--- /dev/null
+++ b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_zh_CN.properties
@@ -0,0 +1,5 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.
\ No newline at end of file
diff --git a/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_zh_HK.properties b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_zh_HK.properties
new file mode 100644
index 00000000..c6a3a0e0
--- /dev/null
+++ b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_zh_HK.properties
@@ -0,0 +1,9 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.errorcode.missing_required_value=您缺失了必須填寫的參數值.
+errorcode.invalid_config_value=您的參數配寘錯誤.
+errorcode.get_topic_info_fail=獲取shard清單失敗.
+errorcode.write_datahub_fail=寫數據失敗.
+errorcode.schema_not_match=數據格式錯誤.
diff --git a/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_zh_TW.properties b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_zh_TW.properties
new file mode 100644
index 00000000..c6a3a0e0
--- /dev/null
+++ b/datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/LocalStrings_zh_TW.properties
@@ -0,0 +1,9 @@
+errorcode.missing_required_value=\u60A8\u7F3A\u5931\u4E86\u5FC5\u987B\u586B\u5199\u7684\u53C2\u6570\u503C.
+errorcode.invalid_config_value=\u60A8\u7684\u53C2\u6570\u914D\u7F6E\u9519\u8BEF.
+errorcode.get_topic_info_fail=\u83B7\u53D6shard\u5217\u8868\u5931\u8D25.
+errorcode.write_datahub_fail=\u5199\u6570\u636E\u5931\u8D25.
+errorcode.schema_not_match=\u6570\u636E\u683C\u5F0F\u9519\u8BEF.errorcode.missing_required_value=您缺失了必須填寫的參數值.
+errorcode.invalid_config_value=您的參數配寘錯誤.
+errorcode.get_topic_info_fail=獲取shard清單失敗.
+errorcode.write_datahub_fail=寫數據失敗.
+errorcode.schema_not_match=數據格式錯誤.
diff --git a/datahubwriter/src/main/resources/job_config_template.json b/datahubwriter/src/main/resources/job_config_template.json
new file mode 100644
index 00000000..8b0b41ae
--- /dev/null
+++ b/datahubwriter/src/main/resources/job_config_template.json
@@ -0,0 +1,14 @@
+{
+ "name": "datahubwriter",
+ "parameter": {
+ "endpoint":"",
+ "accessId": "",
+ "accessKey": "",
+ "project": "",
+ "topic": "",
+ "mode": "random",
+ "shardId": "",
+ "maxCommitSize": 524288,
+ "maxRetryCount": 500
+ }
+}
\ No newline at end of file
diff --git a/datahubwriter/src/main/resources/plugin.json b/datahubwriter/src/main/resources/plugin.json
new file mode 100644
index 00000000..91c17292
--- /dev/null
+++ b/datahubwriter/src/main/resources/plugin.json
@@ -0,0 +1,6 @@
+{
+ "name": "datahubwriter",
+ "class": "com.alibaba.datax.plugin.writer.datahubwriter.DatahubWriter",
+ "description": "datahub writer",
+ "developer": "alibaba"
+}
\ No newline at end of file
diff --git a/doriswriter/doc/doriswriter.md b/doriswriter/doc/doriswriter.md
new file mode 100644
index 00000000..58a688b8
--- /dev/null
+++ b/doriswriter/doc/doriswriter.md
@@ -0,0 +1,181 @@
+# DorisWriter 插件文档
+
+## 1 快速介绍
+DorisWriter支持将大批量数据写入Doris中。
+
+## 2 实现原理
+DorisWriter 通过Doris原生支持Stream load方式导入数据， DorisWriter会将`reader`读取的数据进行缓存在内存中，拼接成Json文本，然后批量导入至Doris。
+
+## 3 功能说明
+
+### 3.1 配置样例
+
+这里是一份从Stream读取数据后导入至Doris的配置文件。
+
+```
+{
+ "job": {
+ "content": [
+ {
+ "reader": {
+ "name": "mysqlreader",
+ "parameter": {
+ "column": ["emp_no", "birth_date", "first_name","last_name","gender","hire_date"],
+ "connection": [
+ {
+ "jdbcUrl": ["jdbc:mysql://localhost:3306/demo"],
+ "table": ["employees_1"]
+ }
+ ],
+ "username": "root",
+ "password": "xxxxx",
+ "where": ""
+ }
+ },
+ "writer": {
+ "name": "doriswriter",
+ "parameter": {
+ "loadUrl": ["172.16.0.13:8030"],
+ "loadProps": {
+ },
+ "column": ["emp_no", "birth_date", "first_name","last_name","gender","hire_date"],
+ "username": "root",
+ "password": "xxxxxx",
+ "postSql": ["select count(1) from all_employees_info"],
+ "preSql": [],
+ "flushInterval":30000,
+ "connection": [
+ {
+ "jdbcUrl": "jdbc:mysql://172.16.0.13:9030/demo",
+ "selectedDatabase": "demo",
+ "table": ["all_employees_info"]
+ }
+ ],
+ "loadProps": {
+ "format": "json",
+ "strip_outer_array": true
+ }
+ }
+ }
+ }
+ ],
+ "setting": {
+ "speed": {
+ "channel": "1"
+ }
+ }
+ }
+}
+```
+
+### 3.2 参数说明
+
+* **jdbcUrl**
+
+ - 描述：Doris 的 JDBC 连接串，用户执行 preSql 或 postSQL。
+ - 必选：是
+ - 默认值：无
+
+* **loadUrl**
+
+ - 描述：作为 Stream Load 的连接目标。格式为 "ip:port"。其中 IP 是 FE 节点 IP，port 是 FE 节点的 http_port。可以填写多个，多个之间使用英文状态的分号隔开:`;`，doriswriter 将以轮询的方式访问。
+ - 必选：是
+ - 默认值：无
+
+* **username**
+
+ - 描述：访问Doris数据库的用户名
+ - 必选：是
+ - 默认值：无
+
+* **password**
+
+ - 描述：访问Doris数据库的密码
+ - 必选：否
+ - 默认值：空
+
+* **connection.selectedDatabase**
+ - 描述：需要写入的Doris数据库名称。
+ - 必选：是
+ - 默认值：无
+
+* **connection.table**
+ - 描述：需要写入的Doris表名称。
+ - 必选：是
+ - 默认值：无
+
+* **column**
+
+ - 描述：目的表**需要写入数据**的字段，这些字段将作为生成的 Json 数据的字段名。字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。
+ - 必选：是
+ - 默认值：否
+
+* **preSql**
+
+ - 描述：写入数据到目的表前，会先执行这里的标准语句。
+ - 必选：否
+ - 默认值：无
+
+* **postSql**
+
+ - 描述：写入数据到目的表后，会执行这里的标准语句。
+ - 必选：否
+ - 默认值：无
+
+
+* **maxBatchRows**
+
+ - 描述：每批次导入数据的最大行数。和 **batchSize** 共同控制每批次的导入数量。每批次数据达到两个阈值之一，即开始导入这一批次的数据。
+ - 必选：否
+ - 默认值：500000
+
+* **batchSize**
+
+ - 描述：每批次导入数据的最大数据量。和 **maxBatchRows** 共同控制每批次的导入数量。每批次数据达到两个阈值之一，即开始导入这一批次的数据。
+ - 必选：否
+ - 默认值：104857600
+
+* **maxRetries**
+
+ - 描述：每批次导入数据失败后的重试次数。
+ - 必选：否
+ - 默认值：0
+
+* **labelPrefix**
+
+ - 描述：每批次导入任务的 label 前缀。最终的 label 将有 `labelPrefix + UUID` 组成全局唯一的 label，确保数据不会重复导入
+ - 必选：否
+ - 默认值：`datax_doris_writer_`
+
+* **loadProps**
+
+ - 描述：StreamLoad 的请求参数，详情参照StreamLoad介绍页面。[Stream load - Apache Doris](https://doris.apache.org/zh-CN/docs/data-operate/import/import-way/stream-load-manual)
+
+ 这里包括导入的数据格式：format等，导入数据格式默认我们使用csv，支持JSON，具体可以参照下面类型转换部分，也可以参照上面Stream load 官方信息
+
+ - 必选：否
+
+ - 默认值：无
+
+### 类型转换
+
+默认传入的数据均会被转为字符串，并以`\t`作为列分隔符，`\n`作为行分隔符，组成`csv`文件进行StreamLoad导入操作。
+
+默认是csv格式导入，如需更改列分隔符， 则正确配置 `loadProps` 即可：
+
+```json
+"loadProps": {
+ "column_separator": "\\x01",
+ "line_delimiter": "\\x02"
+}
+```
+
+如需更改导入格式为`json`， 则正确配置 `loadProps` 即可：
+```json
+"loadProps": {
+ "format": "json",
+ "strip_outer_array": true
+}
+```
+
+更多信息请参照 Doris 官网：[Stream load - Apache Doris](https://doris.apache.org/zh-CN/docs/data-operate/import/import-way/stream-load-manual)
\ No newline at end of file
diff --git a/doriswriter/doc/mysql2doris.json b/doriswriter/doc/mysql2doris.json
new file mode 100644
index 00000000..6992a2be
--- /dev/null
+++ b/doriswriter/doc/mysql2doris.json
@@ -0,0 +1,46 @@
+{
+ "job": {
+ "content": [
+ {
+ "reader": {
+ "name": "mysqlreader",
+ "parameter": {
+ "column": ["k1", "k2", "k3"],
+ "connection": [
+ {
+ "jdbcUrl": ["jdbc:mysql://192.168.10.10:3306/db1"],
+ "table": ["t1"]
+ }
+ ],
+ "username": "root",
+ "password": "",
+ "where": ""
+ }
+ },
+ "writer": {
+ "name": "doriswriter",
+ "parameter": {
+ "loadUrl": ["192.168.1.1:8030"],
+ "loadProps": {},
+ "database": "db1",
+ "column": ["k1", "k2", "k3"],
+ "username": "root",
+ "password": "",
+ "postSql": [],
+ "preSql": [],
+ "connection": [
+ "jdbcUrl":"jdbc:mysql://192.168.1.1:9030/",
+ "table":["xxx"],
+ "selectedDatabase":"xxxx"
+ ]
+ }
+ }
+ }
+ ],
+ "setting": {
+ "speed": {
+ "channel": "1"
+ }
+ }
+ }
+}
diff --git a/doriswriter/pom.xml b/doriswriter/pom.xml
new file mode 100644
index 00000000..aa1e6ff0
--- /dev/null
+++ b/doriswriter/pom.xml
@@ -0,0 +1,99 @@
+
+
+
+
+ datax-all
+ com.alibaba.datax
+ 0.0.1-SNAPSHOT
+
+ 4.0.0
+ doriswriter
+ doriswriter
+ jar
+
+
+ com.alibaba.datax
+ datax-common
+ ${datax-project-version}
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+ ch.qos.logback
+ logback-classic
+
+
+ com.alibaba.datax
+ plugin-rdbms-util
+ ${datax-project-version}
+
+
+ mysql
+ mysql-connector-java
+ ${mysql.driver.version}
+
+
+ org.apache.httpcomponents
+ httpclient
+ 4.5.13
+
+
+
+
+
+
+ maven-compiler-plugin
+
+ ${jdk-version}
+ ${jdk-version}
+ ${project-sourceEncoding}
+
+
+
+
+ maven-assembly-plugin
+
+
+ src/main/assembly/package.xml
+
+ datax
+
+
+
+ dwzip
+ package
+
+ single
+
+
+
+
+
+
+
diff --git a/doriswriter/src/main/assembly/package.xml b/doriswriter/src/main/assembly/package.xml
new file mode 100644
index 00000000..71596332
--- /dev/null
+++ b/doriswriter/src/main/assembly/package.xml
@@ -0,0 +1,52 @@
+
+
+
+
+
+ dir
+
+ false
+
+
+ src/main/resources
+
+ plugin.json
+ plugin_job_template.json
+
+ plugin/writer/doriswriter
+
+
+ target/
+
+ doriswriter-0.0.1-SNAPSHOT.jar
+
+ plugin/writer/doriswriter
+
+
+
+
+ false
+ plugin/writer/doriswriter/libs
+ runtime
+
+
+
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DelimiterParser.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DelimiterParser.java
new file mode 100644
index 00000000..e84bd7dd
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DelimiterParser.java
@@ -0,0 +1,54 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import com.google.common.base.Strings;
+
+import java.io.StringWriter;
+
+public class DelimiterParser {
+
+ private static final String HEX_STRING = "0123456789ABCDEF";
+
+ public static String parse(String sp, String dSp) throws RuntimeException {
+ if ( Strings.isNullOrEmpty(sp)) {
+ return dSp;
+ }
+ if (!sp.toUpperCase().startsWith("\\X")) {
+ return sp;
+ }
+ String hexStr = sp.substring(2);
+ // check hex str
+ if (hexStr.isEmpty()) {
+ throw new RuntimeException("Failed to parse delimiter: `Hex str is empty`");
+ }
+ if (hexStr.length() % 2 != 0) {
+ throw new RuntimeException("Failed to parse delimiter: `Hex str length error`");
+ }
+ for (char hexChar : hexStr.toUpperCase().toCharArray()) {
+ if (HEX_STRING.indexOf(hexChar) == -1) {
+ throw new RuntimeException("Failed to parse delimiter: `Hex str format error`");
+ }
+ }
+ // transform to separator
+ StringWriter writer = new StringWriter();
+ for (byte b : hexStrToBytes(hexStr)) {
+ writer.append((char) b);
+ }
+ return writer.toString();
+ }
+
+ private static byte[] hexStrToBytes(String hexStr) {
+ String upperHexStr = hexStr.toUpperCase();
+ int length = upperHexStr.length() / 2;
+ char[] hexChars = upperHexStr.toCharArray();
+ byte[] bytes = new byte[length];
+ for (int i = 0; i < length; i++) {
+ int pos = i * 2;
+ bytes[i] = (byte) (charToByte(hexChars[pos]) << 4 | charToByte(hexChars[pos + 1]));
+ }
+ return bytes;
+ }
+
+ private static byte charToByte(char c) {
+ return (byte) HEX_STRING.indexOf(c);
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisBaseCodec.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisBaseCodec.java
new file mode 100644
index 00000000..ee7ded56
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisBaseCodec.java
@@ -0,0 +1,23 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import com.alibaba.datax.common.element.Column;
+
+public class DorisBaseCodec {
+ protected String convertionField( Column col) {
+ if (null == col.getRawData() || Column.Type.NULL == col.getType()) {
+ return null;
+ }
+ if ( Column.Type.BOOL == col.getType()) {
+ return String.valueOf(col.asLong());
+ }
+ if ( Column.Type.BYTES == col.getType()) {
+ byte[] bts = (byte[])col.getRawData();
+ long value = 0;
+ for (int i = 0; i < bts.length; i++) {
+ value += (bts[bts.length - i - 1] & 0xffL) << (8 * i);
+ }
+ return String.valueOf(value);
+ }
+ return col.asString();
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodec.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodec.java
new file mode 100644
index 00000000..a2437a1c
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodec.java
@@ -0,0 +1,10 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import com.alibaba.datax.common.element.Record;
+
+import java.io.Serializable;
+
+public interface DorisCodec extends Serializable {
+
+ String codec( Record row);
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodecFactory.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodecFactory.java
new file mode 100644
index 00000000..22c4b409
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodecFactory.java
@@ -0,0 +1,19 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import java.util.Map;
+
+public class DorisCodecFactory {
+ public DorisCodecFactory (){
+
+ }
+ public static DorisCodec createCodec( Keys writerOptions) {
+ if ( Keys.StreamLoadFormat.CSV.equals(writerOptions.getStreamLoadFormat())) {
+ Map props = writerOptions.getLoadProps();
+ return new DorisCsvCodec (null == props || !props.containsKey("column_separator") ? null : String.valueOf(props.get("column_separator")));
+ }
+ if ( Keys.StreamLoadFormat.JSON.equals(writerOptions.getStreamLoadFormat())) {
+ return new DorisJsonCodec (writerOptions.getColumns());
+ }
+ throw new RuntimeException("Failed to create row serializer, unsupported `format` from stream load properties.");
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCsvCodec.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCsvCodec.java
new file mode 100644
index 00000000..518aa304
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCsvCodec.java
@@ -0,0 +1,27 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import com.alibaba.datax.common.element.Record;
+
+public class DorisCsvCodec extends DorisBaseCodec implements DorisCodec {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String columnSeparator;
+
+ public DorisCsvCodec ( String sp) {
+ this.columnSeparator = DelimiterParser.parse(sp, "\t");
+ }
+
+ @Override
+ public String codec( Record row) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < row.getColumnNumber(); i++) {
+ String value = convertionField(row.getColumn(i));
+ sb.append(null == value ? "\\N" : value);
+ if (i < row.getColumnNumber() - 1) {
+ sb.append(columnSeparator);
+ }
+ }
+ return sb.toString();
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonCodec.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonCodec.java
new file mode 100644
index 00000000..68abd9eb
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonCodec.java
@@ -0,0 +1,33 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.fastjson2.JSON;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class DorisJsonCodec extends DorisBaseCodec implements DorisCodec {
+
+ private static final long serialVersionUID = 1L;
+
+ private final List fieldNames;
+
+ public DorisJsonCodec ( List fieldNames) {
+ this.fieldNames = fieldNames;
+ }
+
+ @Override
+ public String codec( Record row) {
+ if (null == fieldNames) {
+ return "";
+ }
+ Map rowMap = new HashMap<> (fieldNames.size());
+ int idx = 0;
+ for (String fieldName : fieldNames) {
+ rowMap.put(fieldName, convertionField(row.getColumn(idx)));
+ idx++;
+ }
+ return JSON.toJSONString(rowMap);
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisStreamLoadObserver.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisStreamLoadObserver.java
new file mode 100644
index 00000000..6f7e9a5a
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisStreamLoadObserver.java
@@ -0,0 +1,236 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import com.alibaba.fastjson2.JSON;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultRedirectStrategy;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public class DorisStreamLoadObserver {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisStreamLoadObserver.class);
+
+ private Keys options;
+
+ private long pos;
+ private static final String RESULT_FAILED = "Fail";
+ private static final String RESULT_LABEL_EXISTED = "Label Already Exists";
+ private static final String LAEBL_STATE_VISIBLE = "VISIBLE";
+ private static final String LAEBL_STATE_COMMITTED = "COMMITTED";
+ private static final String RESULT_LABEL_PREPARE = "PREPARE";
+ private static final String RESULT_LABEL_ABORTED = "ABORTED";
+ private static final String RESULT_LABEL_UNKNOWN = "UNKNOWN";
+
+
+ public DorisStreamLoadObserver ( Keys options){
+ this.options = options;
+ }
+
+ public void streamLoad(WriterTuple data) throws Exception {
+ String host = getLoadHost();
+ if(host == null){
+ throw new IOException ("load_url cannot be empty, or the host cannot connect.Please check your configuration.");
+ }
+ String loadUrl = new StringBuilder(host)
+ .append("/api/")
+ .append(options.getDatabase())
+ .append("/")
+ .append(options.getTable())
+ .append("/_stream_load")
+ .toString();
+ LOG.info("Start to join batch data: rows[{}] bytes[{}] label[{}].", data.getRows().size(), data.getBytes(), data.getLabel());
+ Map loadResult = put(loadUrl, data.getLabel(), addRows(data.getRows(), data.getBytes().intValue()));
+ LOG.info("StreamLoad response :{}",JSON.toJSONString(loadResult));
+ final String keyStatus = "Status";
+ if (null == loadResult || !loadResult.containsKey(keyStatus)) {
+ throw new IOException("Unable to flush data to Doris: unknown result status.");
+ }
+ LOG.debug("StreamLoad response:{}",JSON.toJSONString(loadResult));
+ if (RESULT_FAILED.equals(loadResult.get(keyStatus))) {
+ throw new IOException(
+ new StringBuilder("Failed to flush data to Doris.\n").append(JSON.toJSONString(loadResult)).toString()
+ );
+ } else if (RESULT_LABEL_EXISTED.equals(loadResult.get(keyStatus))) {
+ LOG.debug("StreamLoad response:{}",JSON.toJSONString(loadResult));
+ checkStreamLoadState(host, data.getLabel());
+ }
+ }
+
+ private void checkStreamLoadState(String host, String label) throws IOException {
+ int idx = 0;
+ while(true) {
+ try {
+ TimeUnit.SECONDS.sleep(Math.min(++idx, 5));
+ } catch (InterruptedException ex) {
+ break;
+ }
+ try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
+ HttpGet httpGet = new HttpGet(new StringBuilder(host).append("/api/").append(options.getDatabase()).append("/get_load_state?label=").append(label).toString());
+ httpGet.setHeader("Authorization", getBasicAuthHeader(options.getUsername(), options.getPassword()));
+ httpGet.setHeader("Connection", "close");
+
+ try (CloseableHttpResponse resp = httpclient.execute(httpGet)) {
+ HttpEntity respEntity = getHttpEntity(resp);
+ if (respEntity == null) {
+ throw new IOException(String.format("Failed to flush data to Doris, Error " +
+ "could not get the final state of label[%s].\n", label), null);
+ }
+ Map result = (Map)JSON.parse(EntityUtils.toString(respEntity));
+ String labelState = (String)result.get("state");
+ if (null == labelState) {
+ throw new IOException(String.format("Failed to flush data to Doris, Error " +
+ "could not get the final state of label[%s]. response[%s]\n", label, EntityUtils.toString(respEntity)), null);
+ }
+ LOG.info(String.format("Checking label[%s] state[%s]\n", label, labelState));
+ switch(labelState) {
+ case LAEBL_STATE_VISIBLE:
+ case LAEBL_STATE_COMMITTED:
+ return;
+ case RESULT_LABEL_PREPARE:
+ continue;
+ case RESULT_LABEL_ABORTED:
+ throw new DorisWriterExcetion (String.format("Failed to flush data to Doris, Error " +
+ "label[%s] state[%s]\n", label, labelState), null, true);
+ case RESULT_LABEL_UNKNOWN:
+ default:
+ throw new IOException(String.format("Failed to flush data to Doris, Error " +
+ "label[%s] state[%s]\n", label, labelState), null);
+ }
+ }
+ }
+ }
+ }
+
+ private byte[] addRows(List rows, int totalBytes) {
+ if (Keys.StreamLoadFormat.CSV.equals(options.getStreamLoadFormat())) {
+ Map props = (options.getLoadProps() == null ? new HashMap<> () : options.getLoadProps());
+ byte[] lineDelimiter = DelimiterParser.parse((String)props.get("line_delimiter"), "\n").getBytes(StandardCharsets.UTF_8);
+ ByteBuffer bos = ByteBuffer.allocate(totalBytes + rows.size() * lineDelimiter.length);
+ for (byte[] row : rows) {
+ bos.put(row);
+ bos.put(lineDelimiter);
+ }
+ return bos.array();
+ }
+
+ if (Keys.StreamLoadFormat.JSON.equals(options.getStreamLoadFormat())) {
+ ByteBuffer bos = ByteBuffer.allocate(totalBytes + (rows.isEmpty() ? 2 : rows.size() + 1));
+ bos.put("[".getBytes(StandardCharsets.UTF_8));
+ byte[] jsonDelimiter = ",".getBytes(StandardCharsets.UTF_8);
+ boolean isFirstElement = true;
+ for (byte[] row : rows) {
+ if (!isFirstElement) {
+ bos.put(jsonDelimiter);
+ }
+ bos.put(row);
+ isFirstElement = false;
+ }
+ bos.put("]".getBytes(StandardCharsets.UTF_8));
+ return bos.array();
+ }
+ throw new RuntimeException("Failed to join rows data, unsupported `format` from stream load properties:");
+ }
+ private Map put(String loadUrl, String label, byte[] data) throws IOException {
+ LOG.info(String.format("Executing stream load to: '%s', size: '%s'", loadUrl, data.length));
+ final HttpClientBuilder httpClientBuilder = HttpClients.custom()
+ .setRedirectStrategy(new DefaultRedirectStrategy () {
+ @Override
+ protected boolean isRedirectable(String method) {
+ return true;
+ }
+ });
+ try ( CloseableHttpClient httpclient = httpClientBuilder.build()) {
+ HttpPut httpPut = new HttpPut(loadUrl);
+ httpPut.removeHeaders(HttpHeaders.CONTENT_LENGTH);
+ httpPut.removeHeaders(HttpHeaders.TRANSFER_ENCODING);
+ List cols = options.getColumns();
+ if (null != cols && !cols.isEmpty() && Keys.StreamLoadFormat.CSV.equals(options.getStreamLoadFormat())) {
+ httpPut.setHeader("columns", String.join(",", cols.stream().map(f -> String.format("`%s`", f)).collect(Collectors.toList())));
+ }
+ if (null != options.getLoadProps()) {
+ for (Map.Entry entry : options.getLoadProps().entrySet()) {
+ httpPut.setHeader(entry.getKey(), String.valueOf(entry.getValue()));
+ }
+ }
+ httpPut.setHeader("Expect", "100-continue");
+ httpPut.setHeader("label", label);
+ httpPut.setHeader("two_phase_commit", "false");
+ httpPut.setHeader("Authorization", getBasicAuthHeader(options.getUsername(), options.getPassword()));
+ httpPut.setEntity(new ByteArrayEntity(data));
+ httpPut.setConfig(RequestConfig.custom().setRedirectsEnabled(true).build());
+ try ( CloseableHttpResponse resp = httpclient.execute(httpPut)) {
+ HttpEntity respEntity = getHttpEntity(resp);
+ if (respEntity == null)
+ return null;
+ return (Map)JSON.parse(EntityUtils.toString(respEntity));
+ }
+ }
+ }
+
+ private String getBasicAuthHeader(String username, String password) {
+ String auth = username + ":" + password;
+ byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
+ return new StringBuilder("Basic ").append(new String(encodedAuth)).toString();
+ }
+
+ private HttpEntity getHttpEntity(CloseableHttpResponse resp) {
+ int code = resp.getStatusLine().getStatusCode();
+ if (200 != code) {
+ LOG.warn("Request failed with code:{}", code);
+ return null;
+ }
+ HttpEntity respEntity = resp.getEntity();
+ if (null == respEntity) {
+ LOG.warn("Request failed with empty response.");
+ return null;
+ }
+ return respEntity;
+ }
+
+ private String getLoadHost() {
+ List hostList = options.getLoadUrlList();
+ long tmp = pos + hostList.size();
+ for (; pos < tmp; pos++) {
+ String host = new StringBuilder("http://").append(hostList.get((int) (pos % hostList.size()))).toString();
+ if (checkConnection(host)) {
+ return host;
+ }
+ }
+ return null;
+ }
+
+ private boolean checkConnection(String host) {
+ try {
+ URL url = new URL(host);
+ HttpURLConnection co = (HttpURLConnection) url.openConnection();
+ co.setConnectTimeout(5000);
+ co.connect();
+ co.disconnect();
+ return true;
+ } catch (Exception e1) {
+ e1.printStackTrace();
+ return false;
+ }
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisUtil.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisUtil.java
new file mode 100644
index 00000000..5f5a6f34
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisUtil.java
@@ -0,0 +1,105 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import com.alibaba.datax.plugin.rdbms.util.DBUtil;
+import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
+import com.alibaba.datax.plugin.rdbms.util.RdbmsException;
+import com.alibaba.datax.plugin.rdbms.writer.Constant;
+import com.alibaba.druid.sql.parser.ParserException;
+import com.google.common.base.Strings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * jdbc util
+ */
+public class DorisUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisUtil.class);
+
+ private DorisUtil() {}
+
+ public static List getDorisTableColumns( Connection conn, String databaseName, String tableName) {
+ String currentSql = String.format("SELECT COLUMN_NAME FROM `information_schema`.`COLUMNS` WHERE `TABLE_SCHEMA` = '%s' AND `TABLE_NAME` = '%s' ORDER BY `ORDINAL_POSITION` ASC;", databaseName, tableName);
+ List columns = new ArrayList<> ();
+ ResultSet rs = null;
+ try {
+ rs = DBUtil.query(conn, currentSql);
+ while (DBUtil.asyncResultSetNext(rs)) {
+ String colName = rs.getString("COLUMN_NAME");
+ columns.add(colName);
+ }
+ return columns;
+ } catch (Exception e) {
+ throw RdbmsException.asQueryException(DataBaseType.MySql, e, currentSql, null, null);
+ } finally {
+ DBUtil.closeDBResources(rs, null, null);
+ }
+ }
+
+ public static List renderPreOrPostSqls(List preOrPostSqls, String tableName) {
+ if (null == preOrPostSqls) {
+ return Collections.emptyList();
+ }
+ List renderedSqls = new ArrayList<>();
+ for (String sql : preOrPostSqls) {
+ if (! Strings.isNullOrEmpty(sql)) {
+ renderedSqls.add(sql.replace(Constant.TABLE_NAME_PLACEHOLDER, tableName));
+ }
+ }
+ return renderedSqls;
+ }
+
+ public static void executeSqls(Connection conn, List sqls) {
+ Statement stmt = null;
+ String currentSql = null;
+ try {
+ stmt = conn.createStatement();
+ for (String sql : sqls) {
+ currentSql = sql;
+ DBUtil.executeSqlWithoutResultSet(stmt, sql);
+ }
+ } catch (Exception e) {
+ throw RdbmsException.asQueryException(DataBaseType.MySql, e, currentSql, null, null);
+ } finally {
+ DBUtil.closeDBResources(null, stmt, null);
+ }
+ }
+
+ public static void preCheckPrePareSQL( Keys options) {
+ String table = options.getTable();
+ List preSqls = options.getPreSqlList();
+ List renderedPreSqls = DorisUtil.renderPreOrPostSqls(preSqls, table);
+ if (null != renderedPreSqls && !renderedPreSqls.isEmpty()) {
+ LOG.info("Begin to preCheck preSqls:[{}].", String.join(";", renderedPreSqls));
+ for (String sql : renderedPreSqls) {
+ try {
+ DBUtil.sqlValid(sql, DataBaseType.MySql);
+ } catch ( ParserException e) {
+ throw RdbmsException.asPreSQLParserException(DataBaseType.MySql,e,sql);
+ }
+ }
+ }
+ }
+
+ public static void preCheckPostSQL( Keys options) {
+ String table = options.getTable();
+ List postSqls = options.getPostSqlList();
+ List renderedPostSqls = DorisUtil.renderPreOrPostSqls(postSqls, table);
+ if (null != renderedPostSqls && !renderedPostSqls.isEmpty()) {
+ LOG.info("Begin to preCheck postSqls:[{}].", String.join(";", renderedPostSqls));
+ for(String sql : renderedPostSqls) {
+ try {
+ DBUtil.sqlValid(sql, DataBaseType.MySql);
+ } catch (ParserException e){
+ throw RdbmsException.asPostSQLParserException(DataBaseType.MySql,e,sql);
+ }
+ }
+ }
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriter.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriter.java
new file mode 100644
index 00000000..b44d5440
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriter.java
@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+import com.alibaba.datax.common.spi.Writer;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.rdbms.util.DBUtil;
+import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
+import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * doris data writer
+ */
+public class DorisWriter extends Writer {
+
+ public static class Job extends Writer.Job {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Job.class);
+ private Configuration originalConfig = null;
+ private Keys options;
+
+ @Override
+ public void init() {
+ this.originalConfig = super.getPluginJobConf();
+ options = new Keys (super.getPluginJobConf());
+ options.doPretreatment();
+ }
+
+ @Override
+ public void preCheck(){
+ this.init();
+ DorisUtil.preCheckPrePareSQL(options);
+ DorisUtil.preCheckPostSQL(options);
+ }
+
+ @Override
+ public void prepare() {
+ String username = options.getUsername();
+ String password = options.getPassword();
+ String jdbcUrl = options.getJdbcUrl();
+ List renderedPreSqls = DorisUtil.renderPreOrPostSqls(options.getPreSqlList(), options.getTable());
+ if (null != renderedPreSqls && !renderedPreSqls.isEmpty()) {
+ Connection conn = DBUtil.getConnection(DataBaseType.MySql, jdbcUrl, username, password);
+ LOG.info("Begin to execute preSqls:[{}]. context info:{}.", String.join(";", renderedPreSqls), jdbcUrl);
+ DorisUtil.executeSqls(conn, renderedPreSqls);
+ DBUtil.closeDBResources(null, null, conn);
+ }
+ }
+
+ @Override
+ public List split(int mandatoryNumber) {
+ List configurations = new ArrayList<>(mandatoryNumber);
+ for (int i = 0; i < mandatoryNumber; i++) {
+ configurations.add(originalConfig);
+ }
+ return configurations;
+ }
+
+ @Override
+ public void post() {
+ String username = options.getUsername();
+ String password = options.getPassword();
+ String jdbcUrl = options.getJdbcUrl();
+ List renderedPostSqls = DorisUtil.renderPreOrPostSqls(options.getPostSqlList(), options.getTable());
+ if (null != renderedPostSqls && !renderedPostSqls.isEmpty()) {
+ Connection conn = DBUtil.getConnection(DataBaseType.MySql, jdbcUrl, username, password);
+ LOG.info("Start to execute preSqls:[{}]. context info:{}.", String.join(";", renderedPostSqls), jdbcUrl);
+ DorisUtil.executeSqls(conn, renderedPostSqls);
+ DBUtil.closeDBResources(null, null, conn);
+ }
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+ }
+
+ public static class Task extends Writer.Task {
+ private DorisWriterManager writerManager;
+ private Keys options;
+ private DorisCodec rowCodec;
+
+ @Override
+ public void init() {
+ options = new Keys (super.getPluginJobConf());
+ if (options.isWildcardColumn()) {
+ Connection conn = DBUtil.getConnection(DataBaseType.MySql, options.getJdbcUrl(), options.getUsername(), options.getPassword());
+ List columns = DorisUtil.getDorisTableColumns(conn, options.getDatabase(), options.getTable());
+ options.setInfoCchemaColumns(columns);
+ }
+ writerManager = new DorisWriterManager(options);
+ rowCodec = DorisCodecFactory.createCodec(options);
+ }
+
+ @Override
+ public void prepare() {
+ }
+
+ public void startWrite(RecordReceiver recordReceiver) {
+ try {
+ Record record;
+ while ((record = recordReceiver.getFromReader()) != null) {
+ if (record.getColumnNumber() != options.getColumns().size()) {
+ throw DataXException
+ .asDataXException(
+ DBUtilErrorCode.CONF_ERROR,
+ String.format(
+ "There is an error in the column configuration information. " +
+ "This is because you have configured a task where the number of fields to be read from the source:%s " +
+ "is not equal to the number of fields to be written to the destination table:%s. " +
+ "Please check your configuration and make changes.",
+ record.getColumnNumber(),
+ options.getColumns().size()));
+ }
+ writerManager.writeRecord(rowCodec.codec(record));
+ }
+ } catch (Exception e) {
+ throw DataXException.asDataXException(DBUtilErrorCode.WRITE_DATA_ERROR, e);
+ }
+ }
+
+ @Override
+ public void post() {
+ try {
+ writerManager.close();
+ } catch (Exception e) {
+ throw DataXException.asDataXException(DBUtilErrorCode.WRITE_DATA_ERROR, e);
+ }
+ }
+
+ @Override
+ public void destroy() {}
+
+ @Override
+ public boolean supportFailOver(){
+ return false;
+ }
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterExcetion.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterExcetion.java
new file mode 100644
index 00000000..7797d79f
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterExcetion.java
@@ -0,0 +1,29 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class DorisWriterExcetion extends IOException {
+
+ private final Map response;
+ private boolean reCreateLabel;
+
+ public DorisWriterExcetion ( String message, Map response) {
+ super(message);
+ this.response = response;
+ }
+
+ public DorisWriterExcetion ( String message, Map response, boolean reCreateLabel) {
+ super(message);
+ this.response = response;
+ this.reCreateLabel = reCreateLabel;
+ }
+
+ public Map getFailedResponse() {
+ return response;
+ }
+
+ public boolean needReCreateLabel() {
+ return reCreateLabel;
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterManager.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterManager.java
new file mode 100644
index 00000000..f0ba6b52
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterManager.java
@@ -0,0 +1,192 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import com.google.common.base.Strings;
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+public class DorisWriterManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DorisWriterManager.class);
+
+ private final DorisStreamLoadObserver visitor;
+ private final Keys options;
+ private final List buffer = new ArrayList<> ();
+ private int batchCount = 0;
+ private long batchSize = 0;
+ private volatile boolean closed = false;
+ private volatile Exception flushException;
+ private final LinkedBlockingDeque< WriterTuple > flushQueue;
+ private ScheduledExecutorService scheduler;
+ private ScheduledFuture scheduledFuture;
+
+ public DorisWriterManager( Keys options) {
+ this.options = options;
+ this.visitor = new DorisStreamLoadObserver (options);
+ flushQueue = new LinkedBlockingDeque<>(options.getFlushQueueLength());
+ this.startScheduler();
+ this.startAsyncFlushing();
+ }
+
+ public void startScheduler() {
+ stopScheduler();
+ this.scheduler = Executors.newScheduledThreadPool(1, new BasicThreadFactory.Builder().namingPattern("Doris-interval-flush").daemon(true).build());
+ this.scheduledFuture = this.scheduler.schedule(() -> {
+ synchronized (DorisWriterManager.this) {
+ if (!closed) {
+ try {
+ String label = createBatchLabel();
+ LOG.info(String.format("Doris interval Sinking triggered: label[%s].", label));
+ if (batchCount == 0) {
+ startScheduler();
+ }
+ flush(label, false);
+ } catch (Exception e) {
+ flushException = e;
+ }
+ }
+ }
+ }, options.getFlushInterval(), TimeUnit.MILLISECONDS);
+ }
+
+ public void stopScheduler() {
+ if (this.scheduledFuture != null) {
+ scheduledFuture.cancel(false);
+ this.scheduler.shutdown();
+ }
+ }
+
+ public final synchronized void writeRecord(String record) throws IOException {
+ checkFlushException();
+ try {
+ byte[] bts = record.getBytes(StandardCharsets.UTF_8);
+ buffer.add(bts);
+ batchCount++;
+ batchSize += bts.length;
+ if (batchCount >= options.getBatchRows() || batchSize >= options.getBatchSize()) {
+ String label = createBatchLabel();
+ LOG.debug(String.format("Doris buffer Sinking triggered: rows[%d] label[%s].", batchCount, label));
+ flush(label, false);
+ }
+ } catch (Exception e) {
+ throw new IOException("Writing records to Doris failed.", e);
+ }
+ }
+
+ public synchronized void flush(String label, boolean waitUtilDone) throws Exception {
+ checkFlushException();
+ if (batchCount == 0) {
+ if (waitUtilDone) {
+ waitAsyncFlushingDone();
+ }
+ return;
+ }
+ flushQueue.put(new WriterTuple (label, batchSize, new ArrayList<>(buffer)));
+ if (waitUtilDone) {
+ // wait the last flush
+ waitAsyncFlushingDone();
+ }
+ buffer.clear();
+ batchCount = 0;
+ batchSize = 0;
+ }
+
+ public synchronized void close() {
+ if (!closed) {
+ closed = true;
+ try {
+ String label = createBatchLabel();
+ if (batchCount > 0) LOG.debug(String.format("Doris Sink is about to close: label[%s].", label));
+ flush(label, true);
+ } catch (Exception e) {
+ throw new RuntimeException("Writing records to Doris failed.", e);
+ }
+ }
+ checkFlushException();
+ }
+
+ public String createBatchLabel() {
+ StringBuilder sb = new StringBuilder();
+ if (! Strings.isNullOrEmpty(options.getLabelPrefix())) {
+ sb.append(options.getLabelPrefix());
+ }
+ return sb.append(UUID.randomUUID().toString())
+ .toString();
+ }
+
+ private void startAsyncFlushing() {
+ // start flush thread
+ Thread flushThread = new Thread(new Runnable(){
+ public void run() {
+ while(true) {
+ try {
+ asyncFlush();
+ } catch (Exception e) {
+ flushException = e;
+ }
+ }
+ }
+ });
+ flushThread.setDaemon(true);
+ flushThread.start();
+ }
+
+ private void waitAsyncFlushingDone() throws InterruptedException {
+ // wait previous flushings
+ for (int i = 0; i <= options.getFlushQueueLength(); i++) {
+ flushQueue.put(new WriterTuple ("", 0l, null));
+ }
+ checkFlushException();
+ }
+
+ private void asyncFlush() throws Exception {
+ WriterTuple flushData = flushQueue.take();
+ if (Strings.isNullOrEmpty(flushData.getLabel())) {
+ return;
+ }
+ stopScheduler();
+ LOG.debug(String.format("Async stream load: rows[%d] bytes[%d] label[%s].", flushData.getRows().size(), flushData.getBytes(), flushData.getLabel()));
+ for (int i = 0; i <= options.getMaxRetries(); i++) {
+ try {
+ // flush to Doris with stream load
+ visitor.streamLoad(flushData);
+ LOG.info(String.format("Async stream load finished: label[%s].", flushData.getLabel()));
+ startScheduler();
+ break;
+ } catch (Exception e) {
+ LOG.warn("Failed to flush batch data to Doris, retry times = {}", i, e);
+ if (i >= options.getMaxRetries()) {
+ throw new IOException(e);
+ }
+ if (e instanceof DorisWriterExcetion && (( DorisWriterExcetion )e).needReCreateLabel()) {
+ String newLabel = createBatchLabel();
+ LOG.warn(String.format("Batch label changed from [%s] to [%s]", flushData.getLabel(), newLabel));
+ flushData.setLabel(newLabel);
+ }
+ try {
+ Thread.sleep(1000l * Math.min(i + 1, 10));
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Unable to flush, interrupted while doing another attempt", e);
+ }
+ }
+ }
+ }
+
+ private void checkFlushException() {
+ if (flushException != null) {
+ throw new RuntimeException("Writing records to Doris failed.", flushException);
+ }
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/Keys.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/Keys.java
new file mode 100644
index 00000000..e460e76b
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/Keys.java
@@ -0,0 +1,177 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class Keys implements Serializable {
+
+ private static final long serialVersionUID = 1l;
+ private static final int MAX_RETRIES = 3;
+ private static final int BATCH_ROWS = 500000;
+ private static final long DEFAULT_FLUSH_INTERVAL = 30000;
+
+ private static final String LOAD_PROPS_FORMAT = "format";
+ public enum StreamLoadFormat {
+ CSV, JSON;
+ }
+
+ private static final String USERNAME = "username";
+ private static final String PASSWORD = "password";
+ private static final String DATABASE = "connection[0].selectedDatabase";
+ private static final String TABLE = "connection[0].table[0]";
+ private static final String COLUMN = "column";
+ private static final String PRE_SQL = "preSql";
+ private static final String POST_SQL = "postSql";
+ private static final String JDBC_URL = "connection[0].jdbcUrl";
+ private static final String LABEL_PREFIX = "labelPrefix";
+ private static final String MAX_BATCH_ROWS = "maxBatchRows";
+ private static final String MAX_BATCH_SIZE = "batchSize";
+ private static final String FLUSH_INTERVAL = "flushInterval";
+ private static final String LOAD_URL = "loadUrl";
+ private static final String FLUSH_QUEUE_LENGTH = "flushQueueLength";
+ private static final String LOAD_PROPS = "loadProps";
+
+ private static final String DEFAULT_LABEL_PREFIX = "datax_doris_writer_";
+
+ private static final long DEFAULT_MAX_BATCH_SIZE = 90 * 1024 * 1024; //default 90M
+
+ private final Configuration options;
+
+ private List infoSchemaColumns;
+ private List userSetColumns;
+ private boolean isWildcardColumn;
+
+ public Keys ( Configuration options) {
+ this.options = options;
+ this.userSetColumns = options.getList(COLUMN, String.class).stream().map(str -> str.replace("`", "")).collect(Collectors.toList());
+ if (1 == options.getList(COLUMN, String.class).size() && "*".trim().equals(options.getList(COLUMN, String.class).get(0))) {
+ this.isWildcardColumn = true;
+ }
+ }
+
+ public void doPretreatment() {
+ validateRequired();
+ validateStreamLoadUrl();
+ }
+
+ public String getJdbcUrl() {
+ return options.getString(JDBC_URL);
+ }
+
+ public String getDatabase() {
+ return options.getString(DATABASE);
+ }
+
+ public String getTable() {
+ return options.getString(TABLE);
+ }
+
+ public String getUsername() {
+ return options.getString(USERNAME);
+ }
+
+ public String getPassword() {
+ return options.getString(PASSWORD);
+ }
+
+ public String getLabelPrefix() {
+ String label = options.getString(LABEL_PREFIX);
+ return null == label ? DEFAULT_LABEL_PREFIX : label;
+ }
+
+ public List getLoadUrlList() {
+ return options.getList(LOAD_URL, String.class);
+ }
+
+ public List getColumns() {
+ if (isWildcardColumn) {
+ return this.infoSchemaColumns;
+ }
+ return this.userSetColumns;
+ }
+
+ public boolean isWildcardColumn() {
+ return this.isWildcardColumn;
+ }
+
+ public void setInfoCchemaColumns(List cols) {
+ this.infoSchemaColumns = cols;
+ }
+
+ public List getPreSqlList() {
+ return options.getList(PRE_SQL, String.class);
+ }
+
+ public List getPostSqlList() {
+ return options.getList(POST_SQL, String.class);
+ }
+
+ public Map getLoadProps() {
+ return options.getMap(LOAD_PROPS);
+ }
+
+ public int getMaxRetries() {
+ return MAX_RETRIES;
+ }
+
+ public int getBatchRows() {
+ Integer rows = options.getInt(MAX_BATCH_ROWS);
+ return null == rows ? BATCH_ROWS : rows;
+ }
+
+ public long getBatchSize() {
+ Long size = options.getLong(MAX_BATCH_SIZE);
+ return null == size ? DEFAULT_MAX_BATCH_SIZE : size;
+ }
+
+ public long getFlushInterval() {
+ Long interval = options.getLong(FLUSH_INTERVAL);
+ return null == interval ? DEFAULT_FLUSH_INTERVAL : interval;
+ }
+
+ public int getFlushQueueLength() {
+ Integer len = options.getInt(FLUSH_QUEUE_LENGTH);
+ return null == len ? 1 : len;
+ }
+
+ public StreamLoadFormat getStreamLoadFormat() {
+ Map loadProps = getLoadProps();
+ if (null == loadProps) {
+ return StreamLoadFormat.CSV;
+ }
+ if (loadProps.containsKey(LOAD_PROPS_FORMAT)
+ && StreamLoadFormat.JSON.name().equalsIgnoreCase(String.valueOf(loadProps.get(LOAD_PROPS_FORMAT)))) {
+ return StreamLoadFormat.JSON;
+ }
+ return StreamLoadFormat.CSV;
+ }
+
+ private void validateStreamLoadUrl() {
+ List urlList = getLoadUrlList();
+ for (String host : urlList) {
+ if (host.split(":").length < 2) {
+ throw DataXException.asDataXException(DBUtilErrorCode.CONF_ERROR,
+ "The format of loadUrl is not correct, please enter:[`fe_ip:fe_http_ip;fe_ip:fe_http_ip`].");
+ }
+ }
+ }
+
+ private void validateRequired() {
+ final String[] requiredOptionKeys = new String[]{
+ USERNAME,
+ DATABASE,
+ TABLE,
+ COLUMN,
+ LOAD_URL
+ };
+ for (String optionKey : requiredOptionKeys) {
+ options.getNecessaryValue(optionKey, DBUtilErrorCode.REQUIRED_VALUE);
+ }
+ }
+}
diff --git a/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/WriterTuple.java b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/WriterTuple.java
new file mode 100644
index 00000000..32e0b341
--- /dev/null
+++ b/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/WriterTuple.java
@@ -0,0 +1,20 @@
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import java.util.List;
+
+public class WriterTuple {
+ private String label;
+ private Long bytes;
+ private List rows;
+
+ public WriterTuple ( String label, Long bytes, List rows){
+ this.label = label;
+ this.rows = rows;
+ this.bytes = bytes;
+ }
+
+ public String getLabel() { return label; }
+ public void setLabel(String label) { this.label = label; }
+ public Long getBytes() { return bytes; }
+ public List getRows() { return rows; }
+}
diff --git a/doriswriter/src/main/resources/plugin.json b/doriswriter/src/main/resources/plugin.json
new file mode 100644
index 00000000..69dc31a2
--- /dev/null
+++ b/doriswriter/src/main/resources/plugin.json
@@ -0,0 +1,6 @@
+{
+ "name": "doriswriter",
+ "class": "com.alibaba.datax.plugin.writer.doriswriter.DorisWriter",
+ "description": "apache doris writer plugin",
+ "developer": "apche doris"
+}
diff --git a/doriswriter/src/main/resources/plugin_job_template.json b/doriswriter/src/main/resources/plugin_job_template.json
new file mode 100644
index 00000000..0187e539
--- /dev/null
+++ b/doriswriter/src/main/resources/plugin_job_template.json
@@ -0,0 +1,20 @@
+{
+ "name": "doriswriter",
+ "parameter": {
+ "username": "",
+ "password": "",
+ "column": [],
+ "preSql": [],
+ "postSql": [],
+ "beLoadUrl": [],
+ "loadUrl": [],
+ "loadProps": {},
+ "connection": [
+ {
+ "jdbcUrl": "",
+ "selectedDatabase": "",
+ "table": []
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/drdsreader/doc/drdsreader.md b/drdsreader/doc/drdsreader.md
index 25df9200..c54e6bd1 100644
--- a/drdsreader/doc/drdsreader.md
+++ b/drdsreader/doc/drdsreader.md
@@ -50,7 +50,7 @@ DRDS的插件目前DataX只适配了Mysql引擎的场景，DRDS对于DataX而言
// 数据库连接密码
"password": "root",
"column": [
- "id"，"name"
+ "id","name"
],
"connection": [
{
diff --git a/elasticsearchwriter/pom.xml b/elasticsearchwriter/pom.xml
index a60dbd88..8699c6e5 100644
--- a/elasticsearchwriter/pom.xml
+++ b/elasticsearchwriter/pom.xml
@@ -35,12 +35,12 @@
io.searchbox
jest-common
- 2.4.0
+ 6.3.1
io.searchbox
jest
- 2.4.0
+ 6.3.1
joda-time
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESClient.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESClient.java
deleted file mode 100644
index 34bb7e54..00000000
--- a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESClient.java
+++ /dev/null
@@ -1,236 +0,0 @@
-package com.alibaba.datax.plugin.writer.elasticsearchwriter;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import io.searchbox.action.Action;
-import io.searchbox.client.JestClient;
-import io.searchbox.client.JestClientFactory;
-import io.searchbox.client.JestResult;
-import io.searchbox.client.config.HttpClientConfig;
-import io.searchbox.client.config.HttpClientConfig.Builder;
-import io.searchbox.core.Bulk;
-import io.searchbox.indices.CreateIndex;
-import io.searchbox.indices.DeleteIndex;
-import io.searchbox.indices.IndicesExists;
-import io.searchbox.indices.aliases.*;
-import io.searchbox.indices.mapping.PutMapping;
-import org.apache.http.HttpHost;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Created by xiongfeng.bxf on 17/2/8.
- */
-public class ESClient {
- private static final Logger log = LoggerFactory.getLogger(ESClient.class);
-
- private JestClient jestClient;
-
- public JestClient getClient() {
- return jestClient;
- }
-
- public void createClient(String endpoint,
- String user,
- String passwd,
- boolean multiThread,
- int readTimeout,
- boolean compression,
- boolean discovery) {
-
- JestClientFactory factory = new JestClientFactory();
- Builder httpClientConfig = new HttpClientConfig
- .Builder(endpoint)
- .setPreemptiveAuth(new HttpHost(endpoint))
- .multiThreaded(multiThread)
- .connTimeout(30000)
- .readTimeout(readTimeout)
- .maxTotalConnection(200)
- .requestCompressionEnabled(compression)
- .discoveryEnabled(discovery)
- .discoveryFrequency(5l, TimeUnit.MINUTES);
-
- if (!("".equals(user) || "".equals(passwd))) {
- httpClientConfig.defaultCredentials(user, passwd);
- }
-
- factory.setHttpClientConfig(httpClientConfig.build());
-
- jestClient = factory.getObject();
- }
-
- public boolean indicesExists(String indexName) throws Exception {
- boolean isIndicesExists = false;
- JestResult rst = jestClient.execute(new IndicesExists.Builder(indexName).build());
- if (rst.isSucceeded()) {
- isIndicesExists = true;
- } else {
- switch (rst.getResponseCode()) {
- case 404:
- isIndicesExists = false;
- break;
- case 401:
- // 无权访问
- default:
- log.warn(rst.getErrorMessage());
- break;
- }
- }
- return isIndicesExists;
- }
-
- public boolean deleteIndex(String indexName) throws Exception {
- log.info("delete index " + indexName);
- if (indicesExists(indexName)) {
- JestResult rst = execute(new DeleteIndex.Builder(indexName).build());
- if (!rst.isSucceeded()) {
- return false;
- }
- } else {
- log.info("index cannot found, skip delete " + indexName);
- }
- return true;
- }
-
- public boolean createIndex(String indexName, String typeName,
- Object mappings, String settings, boolean dynamic) throws Exception {
- JestResult rst = null;
- if (!indicesExists(indexName)) {
- log.info("create index " + indexName);
- rst = jestClient.execute(
- new CreateIndex.Builder(indexName)
- .settings(settings)
- .setParameter("master_timeout", "5m")
- .build()
- );
- //index_already_exists_exception
- if (!rst.isSucceeded()) {
- if (getStatus(rst) == 400) {
- log.info(String.format("index [%s] already exists", indexName));
- return true;
- } else {
- log.error(rst.getErrorMessage());
- return false;
- }
- } else {
- log.info(String.format("create [%s] index success", indexName));
- }
- }
-
- int idx = 0;
- while (idx < 5) {
- if (indicesExists(indexName)) {
- break;
- }
- Thread.sleep(2000);
- idx ++;
- }
- if (idx >= 5) {
- return false;
- }
-
- if (dynamic) {
- log.info("ignore mappings");
- return true;
- }
- log.info("create mappings for " + indexName + " " + mappings);
- rst = jestClient.execute(new PutMapping.Builder(indexName, typeName, mappings)
- .setParameter("master_timeout", "5m").build());
- if (!rst.isSucceeded()) {
- if (getStatus(rst) == 400) {
- log.info(String.format("index [%s] mappings already exists", indexName));
- } else {
- log.error(rst.getErrorMessage());
- return false;
- }
- } else {
- log.info(String.format("index [%s] put mappings success", indexName));
- }
- return true;
- }
-
- public JestResult execute(Action clientRequest) throws Exception {
- JestResult rst = null;
- rst = jestClient.execute(clientRequest);
- if (!rst.isSucceeded()) {
- //log.warn(rst.getErrorMessage());
- }
- return rst;
- }
-
- public Integer getStatus(JestResult rst) {
- JsonObject jsonObject = rst.getJsonObject();
- if (jsonObject.has("status")) {
- return jsonObject.get("status").getAsInt();
- }
- return 600;
- }
-
- public boolean isBulkResult(JestResult rst) {
- JsonObject jsonObject = rst.getJsonObject();
- return jsonObject.has("items");
- }
-
-
- public boolean alias(String indexname, String aliasname, boolean needClean) throws IOException {
- GetAliases getAliases = new GetAliases.Builder().addIndex(aliasname).build();
- AliasMapping addAliasMapping = new AddAliasMapping.Builder(indexname, aliasname).build();
- JestResult rst = jestClient.execute(getAliases);
- log.info(rst.getJsonString());
- List list = new ArrayList();
- if (rst.isSucceeded()) {
- JsonParser jp = new JsonParser();
- JsonObject jo = (JsonObject)jp.parse(rst.getJsonString());
- for(Map.Entry entry : jo.entrySet()){
- String tindex = entry.getKey();
- if (indexname.equals(tindex)) {
- continue;
- }
- AliasMapping m = new RemoveAliasMapping.Builder(tindex, aliasname).build();
- String s = new Gson().toJson(m.getData());
- log.info(s);
- if (needClean) {
- list.add(m);
- }
- }
- }
-
- ModifyAliases modifyAliases = new ModifyAliases.Builder(addAliasMapping).addAlias(list).setParameter("master_timeout", "5m").build();
- rst = jestClient.execute(modifyAliases);
- if (!rst.isSucceeded()) {
- log.error(rst.getErrorMessage());
- return false;
- }
- return true;
- }
-
- public JestResult bulkInsert(Bulk.Builder bulk, int trySize) throws Exception {
- // es_rejected_execution_exception
- // illegal_argument_exception
- // cluster_block_exception
- JestResult rst = null;
- rst = jestClient.execute(bulk.build());
- if (!rst.isSucceeded()) {
- log.warn(rst.getErrorMessage());
- }
- return rst;
- }
-
- /**
- * 关闭JestClient客户端
- *
- */
- public void closeJestClient() {
- if (jestClient != null) {
- jestClient.shutdownClient();
- }
- }
-}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESColumn.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESColumn.java
deleted file mode 100644
index 8990d77c..00000000
--- a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESColumn.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package com.alibaba.datax.plugin.writer.elasticsearchwriter;
-
-/**
- * Created by xiongfeng.bxf on 17/3/2.
- */
-public class ESColumn {
-
- private String name;//: "appkey",
-
- private String type;//": "TEXT",
-
- private String timezone;
-
- private String format;
-
- private Boolean array;
-
- public void setName(String name) {
- this.name = name;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public void setTimeZone(String timezone) {
- this.timezone = timezone;
- }
-
- public void setFormat(String format) {
- this.format = format;
- }
-
- public String getName() {
- return name;
- }
-
- public String getType() {
- return type;
- }
-
- public String getTimezone() {
- return timezone;
- }
-
- public String getFormat() {
- return format;
- }
-
- public void setTimezone(String timezone) {
- this.timezone = timezone;
- }
-
- public Boolean isArray() {
- return array;
- }
-
- public void setArray(Boolean array) {
- this.array = array;
- }
-
- public Boolean getArray() {
- return array;
- }
-}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESWriter.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESWriter.java
deleted file mode 100644
index eb0e9a81..00000000
--- a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESWriter.java
+++ /dev/null
@@ -1,460 +0,0 @@
-package com.alibaba.datax.plugin.writer.elasticsearchwriter;
-
-import com.alibaba.datax.common.element.Column;
-import com.alibaba.datax.common.element.Record;
-import com.alibaba.datax.common.exception.DataXException;
-import com.alibaba.datax.common.plugin.RecordReceiver;
-import com.alibaba.datax.common.spi.Writer;
-import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.datax.common.util.RetryUtil;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONObject;
-import com.alibaba.fastjson.TypeReference;
-import io.searchbox.client.JestResult;
-import io.searchbox.core.Bulk;
-import io.searchbox.core.BulkResult;
-import io.searchbox.core.Index;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URLEncoder;
-import java.util.*;
-import java.util.concurrent.Callable;
-
-public class ESWriter extends Writer {
- private final static String WRITE_COLUMNS = "write_columns";
-
- public static class Job extends Writer.Job {
- private static final Logger log = LoggerFactory.getLogger(Job.class);
-
- private Configuration conf = null;
-
- @Override
- public void init() {
- this.conf = super.getPluginJobConf();
- }
-
- @Override
- public void prepare() {
- /**
- * 注意：此方法仅执行一次。
- * 最佳实践：如果 Job 中有需要进行数据同步之前的处理，可以在此处完成，如果没有必要则可以直接去掉。
- */
- ESClient esClient = new ESClient();
- esClient.createClient(Key.getEndpoint(conf),
- Key.getAccessID(conf),
- Key.getAccessKey(conf),
- false,
- 300000,
- false,
- false);
-
- String indexName = Key.getIndexName(conf);
- String typeName = Key.getTypeName(conf);
- boolean dynamic = Key.getDynamic(conf);
- String mappings = genMappings(typeName);
- String settings = JSONObject.toJSONString(
- Key.getSettings(conf)
- );
- log.info(String.format("index:[%s], type:[%s], mappings:[%s]", indexName, typeName, mappings));
-
- try {
- boolean isIndicesExists = esClient.indicesExists(indexName);
- if (Key.isCleanup(this.conf) && isIndicesExists) {
- esClient.deleteIndex(indexName);
- }
- // 强制创建,内部自动忽略已存在的情况
- if (!esClient.createIndex(indexName, typeName, mappings, settings, dynamic)) {
- throw new IOException("create index or mapping failed");
- }
- } catch (Exception ex) {
- throw DataXException.asDataXException(ESWriterErrorCode.ES_MAPPINGS, ex.toString());
- }
- esClient.closeJestClient();
- }
-
- private String genMappings(String typeName) {
- String mappings = null;
- Map propMap = new HashMap();
- List columnList = new ArrayList();
-
- List column = conf.getList("column");
- if (column != null) {
- for (Object col : column) {
- JSONObject jo = JSONObject.parseObject(col.toString());
- String colName = jo.getString("name");
- String colTypeStr = jo.getString("type");
- if (colTypeStr == null) {
- throw DataXException.asDataXException(ESWriterErrorCode.BAD_CONFIG_VALUE, col.toString() + " column must have type");
- }
- ESFieldType colType = ESFieldType.getESFieldType(colTypeStr);
- if (colType == null) {
- throw DataXException.asDataXException(ESWriterErrorCode.BAD_CONFIG_VALUE, col.toString() + " unsupported type");
- }
-
- ESColumn columnItem = new ESColumn();
-
- if (colName.equals(Key.PRIMARY_KEY_COLUMN_NAME)) {
- // 兼容已有版本
- colType = ESFieldType.ID;
- colTypeStr = "id";
- }
-
- columnItem.setName(colName);
- columnItem.setType(colTypeStr);
-
- if (colType == ESFieldType.ID) {
- columnList.add(columnItem);
- // 如果是id,则properties为空
- continue;
- }
-
- Boolean array = jo.getBoolean("array");
- if (array != null) {
- columnItem.setArray(array);
- }
- Map field = new HashMap();
- field.put("type", colTypeStr);
- //https://www.elastic.co/guide/en/elasticsearch/reference/5.2/breaking_50_mapping_changes.html#_literal_index_literal_property
- // https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_deep_dive_on_doc_values.html#_disabling_doc_values
- field.put("doc_values", jo.getBoolean("doc_values"));
- field.put("ignore_above", jo.getInteger("ignore_above"));
- field.put("index", jo.getBoolean("index"));
-
- switch (colType) {
- case STRING:
- // 兼容string类型,ES5之前版本
- break;
- case KEYWORD:
- // https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-search-speed.html#_warm_up_global_ordinals
- field.put("eager_global_ordinals", jo.getBoolean("eager_global_ordinals"));
- case TEXT:
- field.put("analyzer", jo.getString("analyzer"));
- // 优化disk使用,也同步会提高index性能
- // https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-disk-usage.html
- field.put("norms", jo.getBoolean("norms"));
- field.put("index_options", jo.getBoolean("index_options"));
- break;
- case DATE:
- columnItem.setTimeZone(jo.getString("timezone"));
- columnItem.setFormat(jo.getString("format"));
- // 后面时间会处理为带时区的标准时间,所以不需要给ES指定格式
- /*
- if (jo.getString("format") != null) {
- field.put("format", jo.getString("format"));
- } else {
- //field.put("format", "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd");
- }
- */
- break;
- case GEO_SHAPE:
- field.put("tree", jo.getString("tree"));
- field.put("precision", jo.getString("precision"));
- default:
- break;
- }
- propMap.put(colName, field);
- columnList.add(columnItem);
- }
- }
-
- conf.set(WRITE_COLUMNS, JSON.toJSONString(columnList));
-
- log.info(JSON.toJSONString(columnList));
-
- Map rootMappings = new HashMap();
- Map typeMappings = new HashMap();
- typeMappings.put("properties", propMap);
- rootMappings.put(typeName, typeMappings);
-
- mappings = JSON.toJSONString(rootMappings);
-
- if (mappings == null || "".equals(mappings)) {
- throw DataXException.asDataXException(ESWriterErrorCode.BAD_CONFIG_VALUE, "must have mappings");
- }
-
- return mappings;
- }
-
- @Override
- public List split(int mandatoryNumber) {
- List configurations = new ArrayList(mandatoryNumber);
- for (int i = 0; i < mandatoryNumber; i++) {
- configurations.add(conf);
- }
- return configurations;
- }
-
- @Override
- public void post() {
- ESClient esClient = new ESClient();
- esClient.createClient(Key.getEndpoint(conf),
- Key.getAccessID(conf),
- Key.getAccessKey(conf),
- false,
- 300000,
- false,
- false);
- String alias = Key.getAlias(conf);
- if (!"".equals(alias)) {
- log.info(String.format("alias [%s] to [%s]", alias, Key.getIndexName(conf)));
- try {
- esClient.alias(Key.getIndexName(conf), alias, Key.isNeedCleanAlias(conf));
- } catch (IOException e) {
- throw DataXException.asDataXException(ESWriterErrorCode.ES_ALIAS_MODIFY, e);
- }
- }
- }
-
- @Override
- public void destroy() {
-
- }
- }
-
- public static class Task extends Writer.Task {
-
- private static final Logger log = LoggerFactory.getLogger(Job.class);
-
- private Configuration conf;
-
-
- ESClient esClient = null;
- private List typeList;
- private List columnList;
-
- private int trySize;
- private int batchSize;
- private String index;
- private String type;
- private String splitter;
-
- @Override
- public void init() {
- this.conf = super.getPluginJobConf();
- index = Key.getIndexName(conf);
- type = Key.getTypeName(conf);
-
- trySize = Key.getTrySize(conf);
- batchSize = Key.getBatchSize(conf);
- splitter = Key.getSplitter(conf);
- columnList = JSON.parseObject(this.conf.getString(WRITE_COLUMNS), new TypeReference>() {
- });
-
- typeList = new ArrayList();
-
- for (ESColumn col : columnList) {
- typeList.add(ESFieldType.getESFieldType(col.getType()));
- }
-
- esClient = new ESClient();
- }
-
- @Override
- public void prepare() {
- esClient.createClient(Key.getEndpoint(conf),
- Key.getAccessID(conf),
- Key.getAccessKey(conf),
- Key.isMultiThread(conf),
- Key.getTimeout(conf),
- Key.isCompression(conf),
- Key.isDiscovery(conf));
- }
-
- @Override
- public void startWrite(RecordReceiver recordReceiver) {
- List writerBuffer = new ArrayList(this.batchSize);
- Record record = null;
- long total = 0;
- while ((record = recordReceiver.getFromReader()) != null) {
- writerBuffer.add(record);
- if (writerBuffer.size() >= this.batchSize) {
- total += doBatchInsert(writerBuffer);
- writerBuffer.clear();
- }
- }
-
- if (!writerBuffer.isEmpty()) {
- total += doBatchInsert(writerBuffer);
- writerBuffer.clear();
- }
-
- String msg = String.format("task end, write size :%d", total);
- getTaskPluginCollector().collectMessage("writesize", String.valueOf(total));
- log.info(msg);
- esClient.closeJestClient();
- }
-
- private String getDateStr(ESColumn esColumn, Column column) {
- DateTime date = null;
- DateTimeZone dtz = DateTimeZone.getDefault();
- if (esColumn.getTimezone() != null) {
- // 所有时区参考 http://www.joda.org/joda-time/timezones.html
- dtz = DateTimeZone.forID(esColumn.getTimezone());
- }
- if (column.getType() != Column.Type.DATE && esColumn.getFormat() != null) {
- DateTimeFormatter formatter = DateTimeFormat.forPattern(esColumn.getFormat());
- date = formatter.withZone(dtz).parseDateTime(column.asString());
- return date.toString();
- } else if (column.getType() == Column.Type.DATE) {
- date = new DateTime(column.asLong(), dtz);
- return date.toString();
- } else {
- return column.asString();
- }
- }
-
- private long doBatchInsert(final List writerBuffer) {
- Map data = null;
- final Bulk.Builder bulkaction = new Bulk.Builder().defaultIndex(this.index).defaultType(this.type);
- for (Record record : writerBuffer) {
- data = new HashMap();
- String id = null;
- for (int i = 0; i < record.getColumnNumber(); i++) {
- Column column = record.getColumn(i);
- String columnName = columnList.get(i).getName();
- ESFieldType columnType = typeList.get(i);
- //如果是数组类型，那它传入的必是字符串类型
- if (columnList.get(i).isArray() != null && columnList.get(i).isArray()) {
- String[] dataList = column.asString().split(splitter);
- if (!columnType.equals(ESFieldType.DATE)) {
- data.put(columnName, dataList);
- } else {
- for (int pos = 0; pos < dataList.length; pos++) {
- dataList[pos] = getDateStr(columnList.get(i), column);
- }
- data.put(columnName, dataList);
- }
- } else {
- switch (columnType) {
- case ID:
- if (id != null) {
- id += record.getColumn(i).asString();
- } else {
- id = record.getColumn(i).asString();
- }
- break;
- case DATE:
- try {
- String dateStr = getDateStr(columnList.get(i), column);
- data.put(columnName, dateStr);
- } catch (Exception e) {
- getTaskPluginCollector().collectDirtyRecord(record, String.format("时间类型解析失败 [%s:%s] exception: %s", columnName, column.toString(), e.toString()));
- }
- break;
- case KEYWORD:
- case STRING:
- case TEXT:
- case IP:
- case GEO_POINT:
- data.put(columnName, column.asString());
- break;
- case BOOLEAN:
- data.put(columnName, column.asBoolean());
- break;
- case BYTE:
- case BINARY:
- data.put(columnName, column.asBytes());
- break;
- case LONG:
- data.put(columnName, column.asLong());
- break;
- case INTEGER:
- data.put(columnName, column.asBigInteger());
- break;
- case SHORT:
- data.put(columnName, column.asBigInteger());
- break;
- case FLOAT:
- case DOUBLE:
- data.put(columnName, column.asDouble());
- break;
- case NESTED:
- case OBJECT:
- case GEO_SHAPE:
- data.put(columnName, JSON.parse(column.asString()));
- break;
- default:
- getTaskPluginCollector().collectDirtyRecord(record, "类型错误:不支持的类型:" + columnType + " " + columnName);
- }
- }
- }
-
- if (id == null) {
- //id = UUID.randomUUID().toString();
- bulkaction.addAction(new Index.Builder(data).build());
- } else {
- bulkaction.addAction(new Index.Builder(data).id(id).build());
- }
- }
-
- try {
- return RetryUtil.executeWithRetry(new Callable() {
- @Override
- public Integer call() throws Exception {
- JestResult jestResult = esClient.bulkInsert(bulkaction, 1);
- if (jestResult.isSucceeded()) {
- return writerBuffer.size();
- }
-
- String msg = String.format("response code: [%d] error :[%s]", jestResult.getResponseCode(), jestResult.getErrorMessage());
- log.warn(msg);
- if (esClient.isBulkResult(jestResult)) {
- BulkResult brst = (BulkResult) jestResult;
- List failedItems = brst.getFailedItems();
- for (BulkResult.BulkResultItem item : failedItems) {
- if (item.status != 400) {
- // 400 BAD_REQUEST 如果非数据异常,请求异常,则不允许忽略
- throw DataXException.asDataXException(ESWriterErrorCode.ES_INDEX_INSERT, String.format("status:[%d], error: %s", item.status, item.error));
- } else {
- // 如果用户选择不忽略解析错误,则抛异常,默认为忽略
- if (!Key.isIgnoreParseError(conf)) {
- throw DataXException.asDataXException(ESWriterErrorCode.ES_INDEX_INSERT, String.format("status:[%d], error: %s, config not ignoreParseError so throw this error", item.status, item.error));
- }
- }
- }
-
- List items = brst.getItems();
- for (int idx = 0; idx < items.size(); ++idx) {
- BulkResult.BulkResultItem item = items.get(idx);
- if (item.error != null && !"".equals(item.error)) {
- getTaskPluginCollector().collectDirtyRecord(writerBuffer.get(idx), String.format("status:[%d], error: %s", item.status, item.error));
- }
- }
- return writerBuffer.size() - brst.getFailedItems().size();
- } else {
- Integer status = esClient.getStatus(jestResult);
- switch (status) {
- case 429: //TOO_MANY_REQUESTS
- log.warn("server response too many requests, so auto reduce speed");
- break;
- }
- throw DataXException.asDataXException(ESWriterErrorCode.ES_INDEX_INSERT, jestResult.getErrorMessage());
- }
- }
- }, trySize, 60000L, true);
- } catch (Exception e) {
- if (Key.isIgnoreWriteError(this.conf)) {
- log.warn(String.format("重试[%d]次写入失败，忽略该错误，继续写入!", trySize));
- } else {
- throw DataXException.asDataXException(ESWriterErrorCode.ES_INDEX_INSERT, e);
- }
- }
- return 0;
- }
-
- @Override
- public void post() {
- }
-
- @Override
- public void destroy() {
- esClient.closeJestClient();
- }
- }
-}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESWriterErrorCode.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESWriterErrorCode.java
deleted file mode 100644
index 59dcbd0a..00000000
--- a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESWriterErrorCode.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package com.alibaba.datax.plugin.writer.elasticsearchwriter;
-
-import com.alibaba.datax.common.spi.ErrorCode;
-
-public enum ESWriterErrorCode implements ErrorCode {
- BAD_CONFIG_VALUE("ESWriter-00", "您配置的值不合法."),
- ES_INDEX_DELETE("ESWriter-01", "删除index错误."),
- ES_INDEX_CREATE("ESWriter-02", "创建index错误."),
- ES_MAPPINGS("ESWriter-03", "mappings错误."),
- ES_INDEX_INSERT("ESWriter-04", "插入数据错误."),
- ES_ALIAS_MODIFY("ESWriter-05", "别名修改错误."),
- ;
-
- private final String code;
- private final String description;
-
- ESWriterErrorCode(String code, String description) {
- this.code = code;
- this.description = description;
- }
-
- @Override
- public String getCode() {
- return this.code;
- }
-
- @Override
- public String getDescription() {
- return this.description;
- }
-
- @Override
- public String toString() {
- return String.format("Code:[%s], Description:[%s]. ", this.code,
- this.description);
- }
-}
\ No newline at end of file
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchClient.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchClient.java
new file mode 100644
index 00000000..08486e1f
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchClient.java
@@ -0,0 +1,314 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.writer.elasticsearchwriter.jest.ClusterInfo;
+import com.alibaba.datax.plugin.writer.elasticsearchwriter.jest.ClusterInfoResult;
+import com.alibaba.datax.plugin.writer.elasticsearchwriter.jest.PutMapping7;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONObject;
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import io.searchbox.action.Action;
+import io.searchbox.client.JestClient;
+import io.searchbox.client.JestClientFactory;
+import io.searchbox.client.JestResult;
+import io.searchbox.client.config.HttpClientConfig;
+import io.searchbox.client.config.HttpClientConfig.Builder;
+import io.searchbox.core.Bulk;
+import io.searchbox.indices.CreateIndex;
+import io.searchbox.indices.DeleteIndex;
+import io.searchbox.indices.IndicesExists;
+import io.searchbox.indices.aliases.*;
+import io.searchbox.indices.mapping.GetMapping;
+import io.searchbox.indices.mapping.PutMapping;
+
+import io.searchbox.indices.settings.GetSettings;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Created by xiongfeng.bxf on 17/2/8.
+ */
+public class ElasticSearchClient {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchClient.class);
+
+ private JestClient jestClient;
+ private Configuration conf;
+
+ public JestClient getClient() {
+ return jestClient;
+ }
+
+ public ElasticSearchClient(Configuration conf) {
+ this.conf = conf;
+ String endpoint = Key.getEndpoint(conf);
+ //es是支持集群写入的
+ String[] endpoints = endpoint.split(",");
+ String user = Key.getUsername(conf);
+ String passwd = Key.getPassword(conf);
+ boolean multiThread = Key.isMultiThread(conf);
+ int readTimeout = Key.getTimeout(conf);
+ boolean compression = Key.isCompression(conf);
+ boolean discovery = Key.isDiscovery(conf);
+ String discoveryFilter = Key.getDiscoveryFilter(conf);
+ int totalConnection = this.conf.getInt("maxTotalConnection", 200);
+ JestClientFactory factory = new JestClientFactory();
+ Builder httpClientConfig = new HttpClientConfig
+ .Builder(Arrays.asList(endpoints))
+// .setPreemptiveAuth(new HttpHost(endpoint))
+ .multiThreaded(multiThread)
+ .connTimeout(readTimeout)
+ .readTimeout(readTimeout)
+ .maxTotalConnection(totalConnection)
+ .requestCompressionEnabled(compression)
+ .discoveryEnabled(discovery)
+ .discoveryFrequency(5L, TimeUnit.MINUTES)
+ .discoveryFilter(discoveryFilter);
+ if (!(StringUtils.isBlank(user) || StringUtils.isBlank(passwd))) {
+ // 匿名登录
+ httpClientConfig.defaultCredentials(user, passwd);
+ }
+ factory.setHttpClientConfig(httpClientConfig.build());
+ this.jestClient = factory.getObject();
+ }
+
+ public boolean indicesExists(String indexName) throws Exception {
+ boolean isIndicesExists = false;
+ JestResult rst = execute(new IndicesExists.Builder(indexName).build());
+ if (rst.isSucceeded()) {
+ isIndicesExists = true;
+ } else {
+ LOGGER.warn("IndicesExists got ResponseCode: {} ErrorMessage: {}", rst.getResponseCode(), rst.getErrorMessage());
+ switch (rst.getResponseCode()) {
+ case 404:
+ isIndicesExists = false;
+ break;
+ case 401:
+ // 无权访问
+ default:
+ LOGGER.warn(rst.getErrorMessage());
+ break;
+ }
+ }
+ return isIndicesExists;
+ }
+
+ public boolean deleteIndex(String indexName) throws Exception {
+ LOGGER.info("delete index {}", indexName);
+ if (indicesExists(indexName)) {
+ JestResult rst = execute(new DeleteIndex.Builder(indexName).build());
+ if (!rst.isSucceeded()) {
+ LOGGER.warn("DeleteIndex got ResponseCode: {}, ErrorMessage: {}", rst.getResponseCode(), rst.getErrorMessage());
+ return false;
+ } else {
+ LOGGER.info("delete index {} success", indexName);
+ }
+ } else {
+ LOGGER.info("index cannot found, skip delete index {}", indexName);
+ }
+ return true;
+ }
+
+ public boolean isGreaterOrEqualThan7() throws Exception {
+ try {
+ ClusterInfoResult result = execute(new ClusterInfo.Builder().build());
+ LOGGER.info("ClusterInfoResult: {}", result.getJsonString());
+ return result.isGreaterOrEqualThan7();
+ }catch(Exception e) {
+ LOGGER.warn(e.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * 获取索引的settings
+ * @param indexName 索引名
+ * @return 设置
+ */
+ public String getIndexSettings(String indexName) {
+ GetSettings.Builder builder = new GetSettings.Builder();
+ builder.addIndex(indexName);
+ GetSettings getSettings = builder.build();
+ try {
+ LOGGER.info("begin GetSettings for index: {}", indexName);
+ JestResult result = this.execute(getSettings);
+ return result.getJsonString();
+ } catch (Exception e) {
+ String message = "GetSettings for index error: " + e.getMessage();
+ LOGGER.warn(message, e);
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.ES_GET_SETTINGS, e.getMessage(), e);
+ }
+ }
+
+ public boolean createIndexIfNotExists(String indexName, String typeName,
+ Object mappings, String settings,
+ boolean dynamic, boolean isGreaterOrEqualThan7) throws Exception {
+ JestResult rst;
+ if (!indicesExists(indexName)) {
+ LOGGER.info("create index {}", indexName);
+ rst = execute(
+ new CreateIndex.Builder(indexName)
+ .settings(settings)
+ .setParameter("master_timeout", Key.getMasterTimeout(this.conf))
+ .build()
+ );
+ //index_already_exists_exception
+ if (!rst.isSucceeded()) {
+ LOGGER.warn("CreateIndex got ResponseCode: {}, ErrorMessage: {}", rst.getResponseCode(), rst.getErrorMessage());
+ if (getStatus(rst) == 400) {
+ LOGGER.info(String.format("index {} already exists", indexName));
+ return true;
+ } else {
+ return false;
+ }
+ } else {
+ LOGGER.info("create {} index success", indexName);
+ }
+ }
+
+ if (dynamic) {
+ LOGGER.info("dynamic is true, ignore mappings");
+ return true;
+ }
+ LOGGER.info("create mappings for {} {}", indexName, mappings);
+ //如果大于7.x，mapping的PUT请求URI中不能带type，并且mapping设置中不能带有嵌套结构
+ if (isGreaterOrEqualThan7) {
+ rst = execute(new PutMapping7.Builder(indexName, mappings).
+ setParameter("master_timeout", Key.getMasterTimeout(this.conf)).build());
+ } else {
+ rst = execute(new PutMapping.Builder(indexName, typeName, mappings)
+ .setParameter("master_timeout", Key.getMasterTimeout(this.conf)).build());
+ }
+ if (!rst.isSucceeded()) {
+ LOGGER.error("PutMapping got ResponseCode: {}, ErrorMessage: {}", rst.getResponseCode(), rst.getErrorMessage());
+ return false;
+ } else {
+ LOGGER.info("index {} put mappings success", indexName);
+ }
+ return true;
+ }
+
+ public T execute(Action clientRequest) throws IOException {
+ T rst = jestClient.execute(clientRequest);
+ if (!rst.isSucceeded()) {
+ LOGGER.warn(rst.getJsonString());
+ }
+ return rst;
+ }
+
+ public Integer getStatus(JestResult rst) {
+ JsonObject jsonObject = rst.getJsonObject();
+ if (jsonObject.has("status")) {
+ return jsonObject.get("status").getAsInt();
+ }
+ return 600;
+ }
+
+ public boolean isBulkResult(JestResult rst) {
+ JsonObject jsonObject = rst.getJsonObject();
+ return jsonObject.has("items");
+ }
+
+
+ public boolean alias(String indexname, String aliasname, boolean needClean) throws IOException {
+ GetAliases getAliases = new GetAliases.Builder().addIndex(aliasname).build();
+ AliasMapping addAliasMapping = new AddAliasMapping.Builder(indexname, aliasname).build();
+ JestResult rst = null;
+ List list = new ArrayList();
+ if (needClean) {
+ rst = execute(getAliases);
+ if (rst.isSucceeded()) {
+ JsonParser jp = new JsonParser();
+ JsonObject jo = (JsonObject) jp.parse(rst.getJsonString());
+ for (Map.Entry entry : jo.entrySet()) {
+ String tindex = entry.getKey();
+ if (indexname.equals(tindex)) {
+ continue;
+ }
+ AliasMapping m = new RemoveAliasMapping.Builder(tindex, aliasname).build();
+ String s = new Gson().toJson(m.getData());
+ LOGGER.info(s);
+ list.add(m);
+ }
+ }
+ }
+
+ ModifyAliases modifyAliases = new ModifyAliases.Builder(addAliasMapping).addAlias(list).setParameter("master_timeout", Key.getMasterTimeout(this.conf)).build();
+ rst = execute(modifyAliases);
+ if (!rst.isSucceeded()) {
+ LOGGER.error(rst.getErrorMessage());
+ throw new IOException(rst.getErrorMessage());
+ }
+ return true;
+ }
+
+ /**
+ * 获取index的mapping
+ */
+ public String getIndexMapping(String indexName) {
+ GetMapping.Builder builder = new GetMapping.Builder();
+ builder.addIndex(indexName);
+ GetMapping getMapping = builder.build();
+ try {
+ LOGGER.info("begin GetMapping for index: {}", indexName);
+ JestResult result = this.execute(getMapping);
+ return result.getJsonString();
+ } catch (Exception e) {
+ String message = "GetMapping for index error: " + e.getMessage();
+ LOGGER.warn(message, e);
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.ES_MAPPINGS, e.getMessage(), e);
+ }
+ }
+
+ public String getMappingForIndexType(String indexName, String typeName) {
+ String indexMapping = this.getIndexMapping(indexName);
+ JSONObject indexMappingInJson = JSON.parseObject(indexMapping);
+ List paths = Arrays.asList(indexName, "mappings");
+ JSONObject properties = JsonPathUtil.getJsonObject(paths, indexMappingInJson);
+ JSONObject propertiesParent = properties;
+ if (StringUtils.isNotBlank(typeName) && properties.containsKey(typeName)) {
+ propertiesParent = (JSONObject) properties.get(typeName);
+ }
+ JSONObject mapping = (JSONObject) propertiesParent.get("properties");
+ return JSON.toJSONString(mapping);
+ }
+
+ public JestResult bulkInsert(Bulk.Builder bulk) throws Exception {
+ // es_rejected_execution_exception
+ // illegal_argument_exception
+ // cluster_block_exception
+ JestResult rst = null;
+ rst = execute(bulk.build());
+ if (!rst.isSucceeded()) {
+ LOGGER.warn(rst.getErrorMessage());
+ }
+ return rst;
+ }
+
+ /**
+ * 关闭JestClient客户端
+ *
+ */
+ public void closeJestClient() {
+ if (jestClient != null) {
+ try {
+ // jestClient.shutdownClient();
+ jestClient.close();
+ } catch (IOException e) {
+ LOGGER.warn("ignore error: ", e.getMessage());
+ }
+
+ }
+ }
+}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchColumn.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchColumn.java
new file mode 100644
index 00000000..a27b15b2
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchColumn.java
@@ -0,0 +1,126 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter;
+
+import java.util.List;
+
+/**
+ * Created by xiongfeng.bxf on 17/3/2.
+ */
+public class ElasticSearchColumn {
+
+ private String name;//: "appkey",
+
+ private String type;//": "TEXT",
+
+ private String timezone;
+
+ /**
+ * 源头数据格式化处理，datax做的事情
+ */
+ private String format;
+
+ /**
+ * 目标端格式化，es原生支持的格式
+ */
+ private String dstFormat;
+
+ private boolean array;
+
+ /**
+ * 是否使用目标端(ES原生)数组类型
+ *
+ * 默认是false
+ */
+ private boolean dstArray = false;
+
+ private boolean jsonArray;
+
+ private boolean origin;
+
+ private List combineFields;
+
+ private String combineFieldsValueSeparator = "-";
+
+ public String getCombineFieldsValueSeparator() {
+ return combineFieldsValueSeparator;
+ }
+
+ public void setCombineFieldsValueSeparator(String combineFieldsValueSeparator) {
+ this.combineFieldsValueSeparator = combineFieldsValueSeparator;
+ }
+
+ public List getCombineFields() {
+ return combineFields;
+ }
+
+ public void setCombineFields(List combineFields) {
+ this.combineFields = combineFields;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public void setTimeZone(String timezone) {
+ this.timezone = timezone;
+ }
+
+ public void setFormat(String format) {
+ this.format = format;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public boolean isOrigin() { return origin; }
+
+ public void setOrigin(boolean origin) { this.origin = origin; }
+
+ public String getTimezone() {
+ return timezone;
+ }
+
+ public String getFormat() {
+ return format;
+ }
+
+ public void setTimezone(String timezone) {
+ this.timezone = timezone;
+ }
+
+ public boolean isArray() {
+ return array;
+ }
+
+ public void setArray(boolean array) {
+ this.array = array;
+ }
+
+ public boolean isJsonArray() {return jsonArray;}
+
+ public void setJsonArray(boolean jsonArray) {this.jsonArray = jsonArray;}
+
+ public String getDstFormat() {
+ return dstFormat;
+ }
+
+ public void setDstFormat(String dstFormat) {
+ this.dstFormat = dstFormat;
+ }
+
+ public boolean isDstArray() {
+ return dstArray;
+ }
+
+ public void setDstArray(boolean dstArray) {
+ this.dstArray = dstArray;
+ }
+}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESFieldType.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchFieldType.java
similarity index 73%
rename from elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESFieldType.java
rename to elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchFieldType.java
index 14b09689..22c3ee6b 100644
--- a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESFieldType.java
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchFieldType.java
@@ -3,8 +3,11 @@ package com.alibaba.datax.plugin.writer.elasticsearchwriter;
/**
* Created by xiongfeng.bxf on 17/3/1.
*/
-public enum ESFieldType {
+public enum ElasticSearchFieldType {
ID,
+ PARENT,
+ ROUTING,
+ VERSION,
STRING,
TEXT,
KEYWORD,
@@ -24,20 +27,18 @@ public enum ESFieldType {
DATE_RANGE,
GEO_POINT,
GEO_SHAPE,
-
IP,
+ IP_RANGE,
COMPLETION,
TOKEN_COUNT,
-
- ARRAY,
OBJECT,
NESTED;
- public static ESFieldType getESFieldType(String type) {
+ public static ElasticSearchFieldType getESFieldType(String type) {
if (type == null) {
return null;
}
- for (ESFieldType f : ESFieldType.values()) {
+ for (ElasticSearchFieldType f : ElasticSearchFieldType.values()) {
if (f.name().compareTo(type.toUpperCase()) == 0) {
return f;
}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchWriter.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchWriter.java
new file mode 100644
index 00000000..2c8ed2d0
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchWriter.java
@@ -0,0 +1,1117 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter;
+
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+import com.alibaba.datax.common.spi.Writer;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.util.DataXCaseEnvUtil;
+import com.alibaba.datax.common.util.RetryUtil;
+import com.alibaba.datax.plugin.writer.elasticsearchwriter.Key.ActionType;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONArray;
+import com.alibaba.fastjson2.JSONObject;
+import com.alibaba.fastjson2.TypeReference;
+import com.alibaba.fastjson2.JSONWriter;
+import com.google.common.base.Joiner;
+import io.searchbox.client.JestResult;
+import io.searchbox.core.*;
+import io.searchbox.params.Parameters;
+import org.apache.commons.lang3.StringUtils;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.Callable;
+
+public class ElasticSearchWriter extends Writer {
+ private final static String WRITE_COLUMNS = "write_columns";
+
+ public static class Job extends Writer.Job {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Job.class);
+
+ private Configuration conf = null;
+ int retryTimes = 3;
+ long sleepTimeInMilliSecond = 10000L;
+
+ private String settingsCache;
+
+ private void setSettings(String settings) {
+ this.settingsCache = JsonUtil.mergeJsonStr(settings, this.settingsCache);
+ }
+
+ @Override
+ public void init() {
+ this.conf = super.getPluginJobConf();
+ //LOGGER.info("conf:{}", conf);
+ this.retryTimes = this.conf.getInt("retryTimes", 3);
+ this.sleepTimeInMilliSecond = this.conf.getLong("sleepTimeInMilliSecond", 10000L);
+ }
+
+ public List getIncludeSettings() {
+ return this.conf.getList("includeSettingKeys", Arrays.asList("number_of_shards", "number_of_replicas"), String.class);
+ }
+
+ /**
+ * 从es中获取的原始settings转为需要的settings
+ * @param originSettings 原始settings
+ * @return settings
+ */
+ private String convertSettings(String originSettings) {
+ if(StringUtils.isBlank(originSettings)) {
+ return null;
+ }
+ JSONObject jsonObject = JSON.parseObject(originSettings);
+ for(String key : jsonObject.keySet()) {
+ JSONObject settingsObj = jsonObject.getJSONObject(key);
+ if(settingsObj != null) {
+ JSONObject indexObj = settingsObj.getJSONObject("settings");
+ JSONObject settings = indexObj.getJSONObject("index");
+ JSONObject filterSettings = new JSONObject();
+ if(settings != null) {
+ List includeSettings = getIncludeSettings();
+ if(includeSettings != null && includeSettings.size() > 0) {
+ for(String includeSetting : includeSettings) {
+ Object fieldValue = settings.get(includeSetting);
+ if(fieldValue != null) {
+ filterSettings.put(includeSetting, fieldValue);
+ }
+ }
+ return filterSettings.toJSONString();
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void prepare() {
+ /**
+ * 注意：此方法仅执行一次。
+ * 最佳实践：如果 Job 中有需要进行数据同步之前的处理，可以在此处完成，如果没有必要则可以直接去掉。
+ * 对于7.x之后的es版本，取消了index设置type的逻辑，因此在prepare阶段，加入了判断是否为7.x及以上版本
+ * 如果是7.x及以上版本，需要对于index的type做不同的处理
+ * 详见 : https://www.elastic.co/guide/en/elasticsearch/reference/6.8/removal-of-types.html
+ */
+ final ElasticSearchClient esClient = new ElasticSearchClient(this.conf);
+ final String indexName = Key.getIndexName(conf);
+ ActionType actionType = Key.getActionType(conf);
+ final String typeName = Key.getTypeName(conf);
+ final boolean dynamic = Key.getDynamic(conf);
+ final String dstDynamic = Key.getDstDynamic(conf);
+ final String newSettings = JSONObject.toJSONString(Key.getSettings(conf));
+ LOGGER.info("conf settings:{}, settingsCache:{}", newSettings, this.settingsCache);
+ final Integer esVersion = Key.getESVersion(conf);
+ boolean hasId = this.hasID();
+ this.conf.set("hasId", hasId);
+ if (ActionType.UPDATE.equals(actionType) && !hasId && !hasPrimaryKeyInfo()) {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.UPDATE_WITH_ID, "Update mode must specify column type with id or primaryKeyInfo config");
+ }
+
+ try {
+ RetryUtil.executeWithRetry(() -> {
+ boolean isGreaterOrEqualThan7 = esClient.isGreaterOrEqualThan7();
+ if (esVersion != null && esVersion >= 7) {
+ isGreaterOrEqualThan7 = true;
+ }
+ String mappings = genMappings(dstDynamic, typeName, isGreaterOrEqualThan7);
+ conf.set("isGreaterOrEqualThan7", isGreaterOrEqualThan7);
+
+
+ LOGGER.info(String.format("index:[%s], type:[%s], mappings:[%s]", indexName, typeName, mappings));
+ boolean isIndicesExists = esClient.indicesExists(indexName);
+ if (isIndicesExists) {
+ try {
+ // 将原有的mapping打印出来，便于排查问题
+ String oldMappings = esClient.getMappingForIndexType(indexName, typeName);
+ LOGGER.info("the mappings for old index is: {}", oldMappings);
+ } catch (Exception e) {
+ LOGGER.warn("warn message: {}", e.getMessage());
+ }
+ }
+
+ if (Key.isTruncate(conf) && isIndicesExists) {
+ // 备份老的索引中的settings到缓存
+ try {
+ String oldOriginSettings = esClient.getIndexSettings(indexName);
+ if (StringUtils.isNotBlank(oldOriginSettings)) {
+ String includeSettings = convertSettings(oldOriginSettings);
+ LOGGER.info("merge1 settings:{}, settingsCache:{}, includeSettings:{}",
+ oldOriginSettings,
+ this.settingsCache, includeSettings);
+ this.setSettings(includeSettings);
+ }
+ } catch (Exception e) {
+ LOGGER.warn("get old settings fail, indexName:{}", indexName);
+ }
+ esClient.deleteIndex(indexName);
+ }
+
+ // 更新缓存中的settings
+ this.setSettings(newSettings);
+ LOGGER.info("merge2 settings:{}, settingsCache:{}", newSettings, this.settingsCache);
+ // 强制创建,内部自动忽略已存在的情况
+ if (!esClient.createIndexIfNotExists(indexName, typeName, mappings, this.settingsCache, dynamic,
+ isGreaterOrEqualThan7)) {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.ES_MAPPINGS, "");
+ }
+
+ return true;
+ }, DataXCaseEnvUtil.getRetryTimes(this.retryTimes), DataXCaseEnvUtil.getRetryInterval(this.sleepTimeInMilliSecond), DataXCaseEnvUtil.getRetryExponential(false));
+ } catch (Exception ex) {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.ES_MAPPINGS, ex.getMessage(), ex);
+ } finally {
+ try {
+ esClient.closeJestClient();
+ } catch (Exception e) {
+ LOGGER.warn("ignore close jest client error: {}", e.getMessage());
+ }
+ }
+ }
+
+ private boolean hasID() {
+ List column = conf.getList("column");
+ if (column != null) {
+ for (Object col : column) {
+ JSONObject jo = JSONObject.parseObject(col.toString());
+ String colTypeStr = jo.getString("type");
+ ElasticSearchFieldType colType = ElasticSearchFieldType.getESFieldType(colTypeStr);
+ if (ElasticSearchFieldType.ID.equals(colType)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean hasPrimaryKeyInfo() {
+ PrimaryKeyInfo primaryKeyInfo = Key.getPrimaryKeyInfo(this.conf);
+ if (null != primaryKeyInfo && null != primaryKeyInfo.getColumn() && !primaryKeyInfo.getColumn().isEmpty()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+
+ private String genMappings(String dstDynamic, String typeName, boolean isGreaterOrEqualThan7) {
+ String mappings;
+ Map propMap = new HashMap();
+ List columnList = new ArrayList();
+ ElasticSearchColumn combineItem = null;
+
+ List column = conf.getList("column");
+ if (column != null) {
+ for (Object col : column) {
+ JSONObject jo = JSONObject.parseObject(col.toString());
+ String colName = jo.getString("name");
+ String colTypeStr = jo.getString("type");
+ if (colTypeStr == null) {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.BAD_CONFIG_VALUE, col.toString() + " column must have type");
+ }
+ ElasticSearchFieldType colType = ElasticSearchFieldType.getESFieldType(colTypeStr);
+ if (colType == null) {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.BAD_CONFIG_VALUE, col.toString() + " unsupported type");
+ }
+
+ ElasticSearchColumn columnItem = new ElasticSearchColumn();
+
+ if (Key.PRIMARY_KEY_COLUMN_NAME.equals(colName)) {
+ // 兼容已有版本
+ colType = ElasticSearchFieldType.ID;
+ colTypeStr = "id";
+ }
+
+ columnItem.setName(colName);
+ columnItem.setType(colTypeStr);
+
+ JSONArray combineFields = jo.getJSONArray("combineFields");
+ if (combineFields != null && !combineFields.isEmpty() && ElasticSearchFieldType.ID.equals(ElasticSearchFieldType.getESFieldType(colTypeStr))) {
+ List fields = new ArrayList();
+ for (Object item : combineFields) {
+ fields.add((String) item);
+ }
+ columnItem.setCombineFields(fields);
+ combineItem = columnItem;
+ }
+
+ String combineFieldsValueSeparator = jo.getString("combineFieldsValueSeparator");
+ if (StringUtils.isNotBlank(combineFieldsValueSeparator)) {
+ columnItem.setCombineFieldsValueSeparator(combineFieldsValueSeparator);
+ }
+
+ // 如果是id，version，routing，不需要创建mapping
+ if (colType == ElasticSearchFieldType.ID || colType == ElasticSearchFieldType.VERSION || colType == ElasticSearchFieldType.ROUTING) {
+ columnList.add(columnItem);
+ continue;
+ }
+
+ // 如果是组合id中的字段，不需要创建mapping
+ // 所以组合id的定义必须要在columns最前面
+ if (combineItem != null && combineItem.getCombineFields().contains(colName)) {
+ columnList.add(columnItem);
+ continue;
+ }
+ columnItem.setDstArray(false);
+ Boolean array = jo.getBoolean("array");
+ if (array != null) {
+ columnItem.setArray(array);
+ Boolean dstArray = jo.getBoolean("dstArray");
+ if(dstArray!=null) {
+ columnItem.setDstArray(dstArray);
+ }
+ } else {
+ columnItem.setArray(false);
+ }
+ Boolean jsonArray = jo.getBoolean("json_array");
+ if (jsonArray != null) {
+ columnItem.setJsonArray(jsonArray);
+ } else {
+ columnItem.setJsonArray(false);
+ }
+ Map field = new HashMap();
+ field.put("type", colTypeStr);
+ //https://www.elastic.co/guide/en/elasticsearch/reference/5.2/breaking_50_mapping_changes.html#_literal_index_literal_property
+ // https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_deep_dive_on_doc_values.html#_disabling_doc_values
+ field.put("doc_values", jo.getBoolean("doc_values"));
+ field.put("ignore_above", jo.getInteger("ignore_above"));
+ field.put("index", jo.getBoolean("index"));
+ switch (colType) {
+ case STRING:
+ // 兼容string类型,ES5之前版本
+ break;
+ case KEYWORD:
+ // https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-search-speed.html#_warm_up_global_ordinals
+ field.put("eager_global_ordinals", jo.getBoolean("eager_global_ordinals"));
+ break;
+ case TEXT:
+ field.put("analyzer", jo.getString("analyzer"));
+ // 优化disk使用,也同步会提高index性能
+ // https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-disk-usage.html
+ field.put("norms", jo.getBoolean("norms"));
+ field.put("index_options", jo.getBoolean("index_options"));
+ if(jo.getString("fields") != null) {
+ field.put("fields", jo.getJSONObject("fields"));
+ }
+ break;
+ case DATE:
+ if (Boolean.TRUE.equals(jo.getBoolean("origin"))) {
+ if (jo.getString("format") != null) {
+ field.put("format", jo.getString("format"));
+ }
+ // es原生format覆盖原先来的format
+ if (jo.getString("dstFormat") != null) {
+ field.put("format", jo.getString("dstFormat"));
+ }
+ if(jo.getBoolean("origin") != null) {
+ columnItem.setOrigin(jo.getBoolean("origin"));
+ }
+ } else {
+ columnItem.setTimeZone(jo.getString("timezone"));
+ columnItem.setFormat(jo.getString("format"));
+ }
+ break;
+ case GEO_SHAPE:
+ field.put("tree", jo.getString("tree"));
+ field.put("precision", jo.getString("precision"));
+ break;
+ case OBJECT:
+ case NESTED:
+ if (jo.getString("dynamic") != null) {
+ field.put("dynamic", jo.getString("dynamic"));
+ }
+ break;
+ default:
+ break;
+ }
+ if (jo.containsKey("other_params")) {
+ field.putAll(jo.getJSONObject("other_params"));
+ }
+ propMap.put(colName, field);
+ columnList.add(columnItem);
+ }
+ }
+
+ long version = System.currentTimeMillis();
+ LOGGER.info("unified version: {}", version);
+ conf.set("version", version);
+ conf.set(WRITE_COLUMNS, JSON.toJSONString(columnList));
+
+ LOGGER.info(JSON.toJSONString(columnList));
+
+ Map rootMappings = new HashMap();
+ Map typeMappings = new HashMap();
+ typeMappings.put("properties", propMap);
+ rootMappings.put(typeName, typeMappings);
+
+ // 7.x以后版本取消了index中关于type的指定，所以mapping的格式只能支持
+ // {
+ // "properties" : {
+ // "abc" : {
+ // "type" : "text"
+ // }
+ // }
+ // }
+ // properties 外不能再嵌套typeName
+
+ if(StringUtils.isNotBlank(dstDynamic)) {
+ typeMappings.put("dynamic", dstDynamic);
+ }
+ if (isGreaterOrEqualThan7) {
+ mappings = JSON.toJSONString(typeMappings);
+ } else {
+ mappings = JSON.toJSONString(rootMappings);
+ }
+ if (StringUtils.isBlank(mappings)) {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.BAD_CONFIG_VALUE, "must have mappings");
+ }
+
+ return mappings;
+ }
+
+ @Override
+ public List split(int mandatoryNumber) {
+ List configurations = new ArrayList(mandatoryNumber);
+ for (int i = 0; i < mandatoryNumber; i++) {
+ configurations.add(this.conf.clone());
+ }
+ return configurations;
+ }
+
+ @Override
+ public void post() {
+ ElasticSearchClient esClient = new ElasticSearchClient(this.conf);
+ String alias = Key.getAlias(conf);
+ if (!"".equals(alias)) {
+ LOGGER.info(String.format("alias [%s] to [%s]", alias, Key.getIndexName(conf)));
+ try {
+ esClient.alias(Key.getIndexName(conf), alias, Key.isNeedCleanAlias(conf));
+ } catch (IOException e) {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.ES_ALIAS_MODIFY, e);
+ }
+ }
+ }
+
+ @Override
+ public void destroy() {
+
+ }
+ }
+
+ public static class Task extends Writer.Task {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(Job.class);
+
+ private Configuration conf;
+
+
+ ElasticSearchClient esClient = null;
+ private List typeList;
+ private List columnList;
+ private List> deleteByConditions;
+
+ private int trySize;
+ private long tryInterval;
+ private int batchSize;
+ private String index;
+ private String type;
+ private String splitter;
+ private ActionType actionType;
+ private ElasticSearchColumn combinedIdColumn;
+ private Map colNameToIndexMap;
+ private Map urlParams;
+ private boolean columnSizeChecked = false;
+ private boolean enableRedundantColumn = false;
+ private boolean enableWriteNull = true;
+ int retryTimes = 3;
+ long sleepTimeInMilliSecond = 10000L;
+ boolean isGreaterOrEqualThan7 = false;
+ private String fieldDelimiter;
+ private boolean hasId;
+ private PrimaryKeyInfo primaryKeyInfo;
+ private boolean hasPrimaryKeyInfo = false;
+ private List esPartitionColumn;
+ private boolean hasEsPartitionColumn = false;
+
+ @Override
+ public void init() {
+ this.conf = super.getPluginJobConf();
+ this.index = Key.getIndexName(conf);
+ this.type = Key.getTypeName(conf);
+ this.trySize = Key.getTrySize(conf);
+ this.tryInterval = Key.getTryInterval(conf);
+ this.batchSize = Key.getBatchSize(conf);
+ this.splitter = Key.getSplitter(conf);
+ this.actionType = Key.getActionType(conf);
+ this.urlParams = Key.getUrlParams(conf);
+ this.enableWriteNull = Key.isEnableNullUpdate(conf);
+ this.retryTimes = this.conf.getInt("retryTimes", 3);
+ this.sleepTimeInMilliSecond = this.conf.getLong("sleepTimeInMilliSecond", 10000L);
+ this.isGreaterOrEqualThan7 = this.conf.getBool("isGreaterOrEqualThan7", false);
+ this.parseDeleteCondition(conf);
+ this.columnList = JSON.parseObject(this.conf.getString(WRITE_COLUMNS), new TypeReference>() {
+ });
+ LOGGER.info("columnList: {}", JSON.toJSONString(columnList));
+ this.hasId = this.conf.getBool("hasId", false);
+ if (hasId) {
+ LOGGER.info("Task has id column, will use it to set _id property");
+ } else {
+ LOGGER.info("Task will use elasticsearch auto generated _id property");
+ }
+ this.fieldDelimiter = Key.getFieldDelimiter(this.conf);
+ this.enableRedundantColumn = this.conf.getBool("enableRedundantColumn", false);
+ this.typeList = new ArrayList();
+ for (ElasticSearchColumn esColumn : this.columnList) {
+ this.typeList.add(ElasticSearchFieldType.getESFieldType(esColumn.getType()));
+ if (esColumn.getCombineFields() != null && esColumn.getCombineFields().size() > 0
+ && ElasticSearchFieldType.getESFieldType(esColumn.getType()).equals(ElasticSearchFieldType.ID)) {
+ combinedIdColumn = esColumn;
+ }
+ }
+ this.primaryKeyInfo = Key.getPrimaryKeyInfo(this.conf);
+ this.esPartitionColumn = Key.getEsPartitionColumn(this.conf);
+ this.colNameToIndexMap = new HashMap(5);
+ this.handleMetaKeys();
+ this.esClient = new ElasticSearchClient(this.conf);
+ }
+
+ private void handleMetaKeys() {
+ if (null != this.primaryKeyInfo && null != this.primaryKeyInfo.getColumn()
+ && !this.primaryKeyInfo.getColumn().isEmpty()) {
+ this.hasPrimaryKeyInfo = true;
+ if (null == this.primaryKeyInfo.getFieldDelimiter()) {
+ if (null != this.fieldDelimiter) {
+ this.primaryKeyInfo.setFieldDelimiter(this.fieldDelimiter);
+ } else {
+ this.primaryKeyInfo.setFieldDelimiter("");
+ }
+ }
+
+ for (String eachPk : this.primaryKeyInfo.getColumn()) {
+ boolean foundKeyInColumn = false;
+ for (int i = 0; i < columnList.size(); i++) {
+ if (StringUtils.equals(eachPk, columnList.get(i).getName())) {
+ this.colNameToIndexMap.put(eachPk, i);
+ foundKeyInColumn = true;
+ break;
+ }
+ }
+ if (!foundKeyInColumn) {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.RECORD_FIELD_NOT_FOUND,
+ "primaryKeyInfo has column not exists in column");
+ }
+ }
+ }
+
+ if (null != this.esPartitionColumn && !this.esPartitionColumn.isEmpty()) {
+ this.hasEsPartitionColumn = true;
+ for (PartitionColumn eachPartitionCol : this.esPartitionColumn) {
+ boolean foundKeyInColumn = false;
+ for (int i = 0; i < columnList.size(); i++) {
+ if (StringUtils.equals(eachPartitionCol.getName(), columnList.get(i).getName())) {
+ this.colNameToIndexMap.put(eachPartitionCol.getName(), i);
+ foundKeyInColumn = true;
+ break;
+ }
+ }
+ if (!foundKeyInColumn) {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.RECORD_FIELD_NOT_FOUND,
+ "esPartitionColumn has column not exists in column");
+ }
+ }
+ }
+ }
+
+ private void parseDeleteCondition(Configuration conf) {
+ List> list = new ArrayList>();
+ String config = Key.getDeleteBy(conf);
+ if (config != null) {
+ JSONArray array = JSON.parseArray(config);
+ for (Object obj : array) {
+ list.add((Map) obj);
+ }
+ deleteByConditions = list;
+ }
+ }
+
+
+ @Override
+ public void prepare() {
+ }
+
+ /**
+ * 示例：{
+ * "deleteBy" : [
+ * {"product_status" : [-1,-2], "sub_status" : -3}，
+ * {"product_status" : -3}
+ * ]
+ * }
+ *
+ * 表示以下两类数据删除：
+ * 1. product_status为-1或-2并且sub_status为-3
+ * 2. product_status为-3
+ *
+ * 注意[{}]返回true
+ * @param record
+ * @return
+ */
+ private boolean isDeleteRecord(Record record) {
+ if (deleteByConditions == null) {
+ return false;
+ }
+
+ Map kv = new HashMap();
+ for (int i = 0; i < record.getColumnNumber(); i++) {
+ Column column = record.getColumn(i);
+ String columnName = columnList.get(i).getName();
+ kv.put(columnName, column.asString());
+ }
+
+ for (Map delCondition : deleteByConditions) {
+ if (meetAllCondition(kv, delCondition)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private boolean meetAllCondition(Map kv, Map delCondition) {
+ for (Map.Entry oneCondition : delCondition.entrySet()) {
+ if (!checkOneCondition(kv, oneCondition)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean checkOneCondition(Map kv, Map.Entry entry) {
+ Object value = kv.get(entry.getKey());
+ if (entry.getValue() instanceof List) {
+ for (Object obj : (List) entry.getValue()) {
+ if (obj.toString().equals(value)) {
+ return true;
+ }
+ }
+ } else {
+ if (value != null && value.equals(entry.getValue().toString())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void startWrite(RecordReceiver recordReceiver) {
+ List writerBuffer = new ArrayList(this.batchSize);
+ Record record = null;
+ while ((record = recordReceiver.getFromReader()) != null) {
+ if (!columnSizeChecked) {
+ boolean isInvalid = true;
+ if (enableRedundantColumn) {
+ isInvalid = this.columnList.size() > record.getColumnNumber();
+ } else {
+ isInvalid = this.columnList.size() != record.getColumnNumber();
+ }
+ if (isInvalid) {
+ String message = String.format(
+ "column number not equal error, reader column size is %s, but the writer column size is %s",
+ record.getColumnNumber(), this.columnList.size());
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.BAD_CONFIG_VALUE, message);
+ }
+ columnSizeChecked = true;
+ }
+ writerBuffer.add(record);
+ if (writerBuffer.size() >= this.batchSize) {
+ this.doBatchInsert(writerBuffer);
+ writerBuffer.clear();
+ }
+ }
+
+ if (!writerBuffer.isEmpty()) {
+ this.doBatchInsert(writerBuffer);
+ writerBuffer.clear();
+ }
+ }
+
+ private String getDateStr(ElasticSearchColumn esColumn, Column column) {
+ // 如果保持原样，就直接返回
+ if (esColumn.isOrigin()) {
+ return column.asString();
+ }
+ DateTime date = null;
+ DateTimeZone dtz = DateTimeZone.getDefault();
+ if (esColumn.getTimezone() != null) {
+ // 所有时区参考 http://www.joda.org/joda-time/timezones.html
+ // TODO：创建一次多处复用
+ dtz = DateTimeZone.forID(esColumn.getTimezone());
+ }
+ if (column.getType() != Column.Type.DATE && esColumn.getFormat() != null) {
+ // TODO：创建一次多处复用
+ DateTimeFormatter formatter = DateTimeFormat.forPattern(esColumn.getFormat());
+ date = formatter.withZone(dtz).parseDateTime(column.asString());
+ return date.toString();
+ } else if (column.getType() == Column.Type.DATE) {
+ if (null == column.getRawData()) {
+ return null;
+ } else {
+ date = new DateTime(column.asLong(), dtz);
+ return date.toString();
+ }
+ } else {
+ return column.asString();
+ }
+ }
+
+ private void doBatchInsert(final List writerBuffer) {
+ Map data = null;
+ Bulk.Builder bulkactionTmp = null;
+ int totalNumber = writerBuffer.size();
+ int dirtyDataNumber = 0;
+ if (this.isGreaterOrEqualThan7) {
+ bulkactionTmp = new Bulk.Builder().defaultIndex(this.index);
+ } else {
+ bulkactionTmp = new Bulk.Builder().defaultIndex(this.index).defaultType(this.type);
+ }
+ final Bulk.Builder bulkaction = bulkactionTmp;
+ // 增加url的参数
+ for (Map.Entry entry : urlParams.entrySet()) {
+ bulkaction.setParameter(entry.getKey(), entry.getValue());
+ }
+ for (Record record : writerBuffer) {
+ data = new HashMap();
+ String id = null;
+ String parent = null;
+ String routing = null;
+ String version = null;
+ String columnName = null;
+ Column column = null;
+ try {
+ for (int i = 0; i < record.getColumnNumber(); i++) {
+ column = record.getColumn(i);
+ columnName = columnList.get(i).getName();
+ // 如果组合id不等于null，需要把相关的字段全部忽略
+ if (combinedIdColumn != null) {
+ if (combinedIdColumn.getCombineFields().contains(columnName)) {
+ continue;
+ }
+ }
+ //如果是json数组，当成对象类型处理
+ ElasticSearchFieldType columnType = columnList.get(i).isJsonArray() ? ElasticSearchFieldType.NESTED : typeList.get(i);
+
+ Boolean dstArray = columnList.get(i).isDstArray();
+
+ //如果是数组类型，那它传入的是字符串类型，也有可能是null
+ if (columnList.get(i).isArray() && null != column.asString()) {
+ String[] dataList = column.asString().split(splitter);
+ if (!columnType.equals(ElasticSearchFieldType.DATE)) {
+ if (dstArray) {
+ try {
+ // 根据客户配置的类型，转换成相应的类型
+ switch (columnType) {
+ case BYTE:
+ case KEYWORD:
+ case TEXT:
+ data.put(columnName, dataList);
+ break;
+ case SHORT:
+ case INTEGER:
+ if (StringUtils.isBlank(column.asString().trim())) {
+ data.put(columnName, null);
+ } else {
+ Integer[] intDataList = new Integer[dataList.length];
+ for (int j = 0; j < dataList.length; j++) {
+ dataList[j] = dataList[j].trim();
+ if (StringUtils.isNotBlank(dataList[j])) {
+ intDataList[j] = Integer.valueOf(dataList[j]);
+ }
+ }
+ data.put(columnName, intDataList);
+ }
+ break;
+ case LONG:
+ if (StringUtils.isBlank(column.asString().trim())) {
+ data.put(columnName, null);
+ } else {
+ Long[] longDataList = new Long[dataList.length];
+ for (int j = 0; j < dataList.length; j++) {
+ dataList[j] = dataList[j].trim();
+ if (StringUtils.isNotBlank(dataList[j])) {
+ longDataList[j] = Long.valueOf(dataList[j]);
+ }
+ }
+ data.put(columnName, longDataList);
+ }
+ break;
+ case FLOAT:
+ case DOUBLE:
+ if (StringUtils.isBlank(column.asString().trim())) {
+ data.put(columnName, null);
+ } else {
+ Double[] doubleDataList = new Double[dataList.length];
+ for (int j = 0; j < dataList.length; j++) {
+ dataList[j] = dataList[j].trim();
+ if (StringUtils.isNotBlank(dataList[j])) {
+ doubleDataList[j] = Double.valueOf(dataList[j]);
+ }
+ }
+ data.put(columnName, doubleDataList);
+ }
+ break;
+ default:
+ data.put(columnName, dataList);
+ break;
+ }
+ } catch (Exception e) {
+ LOGGER.info("脏数据，记录:{}", record.toString());
+ continue;
+ }
+ } else {
+ data.put(columnName, dataList);
+ }
+ } else {
+ data.put(columnName, dataList);
+ }
+ } else {
+ // LOGGER.info("columnType: {} integer: {}", columnType, column.asString());
+ switch (columnType) {
+ case ID:
+ if (id != null) {
+ id += record.getColumn(i).asString();
+ } else {
+ id = record.getColumn(i).asString();
+ }
+ break;
+ case PARENT:
+ if (parent != null) {
+ parent += record.getColumn(i).asString();
+ } else {
+ parent = record.getColumn(i).asString();
+ }
+ break;
+ case ROUTING:
+ if (routing != null) {
+ routing += record.getColumn(i).asString();
+ } else {
+ routing = record.getColumn(i).asString();
+ }
+ break;
+
+ case VERSION:
+ if (version != null) {
+ version += record.getColumn(i).asString();
+ } else {
+ version = record.getColumn(i).asString();
+ }
+ break;
+ case DATE:
+ String dateStr = getDateStr(columnList.get(i), column);
+ data.put(columnName, dateStr);
+ break;
+ case KEYWORD:
+ case STRING:
+ case TEXT:
+ case IP:
+ case GEO_POINT:
+ case IP_RANGE:
+ data.put(columnName, column.asString());
+ break;
+ case BOOLEAN:
+ data.put(columnName, column.asBoolean());
+ break;
+ case BYTE:
+ case BINARY:
+ // json序列化不支持byte类型，es支持的binary类型，必须传入base64的格式
+ data.put(columnName, column.asString());
+ break;
+ case LONG:
+ data.put(columnName, column.asLong());
+ break;
+ case INTEGER:
+ data.put(columnName, column.asLong());
+ break;
+ case SHORT:
+ data.put(columnName, column.asLong());
+ break;
+ case FLOAT:
+ case DOUBLE:
+ data.put(columnName, column.asDouble());
+ break;
+ case GEO_SHAPE:
+ case DATE_RANGE:
+ case INTEGER_RANGE:
+ case FLOAT_RANGE:
+ case LONG_RANGE:
+ case DOUBLE_RANGE:
+ if (null == column.asString()) {
+ data.put(columnName, column.asString());
+ } else {
+ data.put(columnName, JSON.parse(column.asString()));
+ }
+ break;
+ case NESTED:
+ case OBJECT:
+ if (null == column.asString()) {
+ data.put(columnName, column.asString());
+ } else {
+ // 转json格式
+ data.put(columnName, JSON.parse(column.asString()));
+ }
+ break;
+ default:
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.BAD_CONFIG_VALUE, String.format(
+ "Type error: unsupported type %s for column %s", columnType, columnName));
+ }
+ }
+ }
+
+
+ if (this.hasPrimaryKeyInfo) {
+ List idData = new ArrayList();
+ for (String eachCol : this.primaryKeyInfo.getColumn()) {
+ Column recordColumn = record.getColumn(this.colNameToIndexMap.get(eachCol));
+ idData.add(recordColumn.asString());
+ }
+ id = StringUtils.join(idData, this.primaryKeyInfo.getFieldDelimiter());
+ }
+ if (this.hasEsPartitionColumn) {
+ List idData = new ArrayList();
+ for (PartitionColumn eachCol : this.esPartitionColumn) {
+ Column recordColumn = record.getColumn(this.colNameToIndexMap.get(eachCol.getName()));
+ idData.add(recordColumn.asString());
+ }
+ routing = StringUtils.join(idData, "");
+ }
+ } catch (Exception e) {
+ // 脏数据
+ super.getTaskPluginCollector().collectDirtyRecord(record,
+ String.format("parse error for column: %s errorMessage: %s", columnName, e.getMessage()));
+ dirtyDataNumber++;
+ // 处理下一个record
+ continue;
+ }
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("id: {} routing: {} data: {}", id, routing, JSON.toJSONString(data));
+ }
+
+
+ if (isDeleteRecord(record)) {
+ Delete.Builder builder = new Delete.Builder(id);
+ bulkaction.addAction(builder.build());
+ } else {
+ // 使用用户自定义组合唯一键
+ if (combinedIdColumn != null) {
+ try {
+ id = processIDCombineFields(record, combinedIdColumn);
+ // LOGGER.debug("id: {}", id);
+ } catch (Exception e) {
+ // 脏数据
+ super.getTaskPluginCollector().collectDirtyRecord(record,
+ String.format("parse error for column: %s errorMessage: %s", columnName, e.getMessage()));
+ // 处理下一个record
+ dirtyDataNumber++;
+ continue;
+ }
+ }
+ switch (actionType) {
+ case INDEX:
+ // 先进行json序列化，jest client的gson序列化会把等号按照html序列化
+ Index.Builder builder = null;
+ if (this.enableWriteNull) {
+ builder = new Index.Builder(
+ JSONObject.toJSONString(data, JSONWriter.Feature.WriteMapNullValue,
+ JSONWriter.Feature.WriteEnumUsingToString));
+ } else {
+ builder = new Index.Builder(JSONObject.toJSONString(data));
+ }
+ if (id != null) {
+ builder.id(id);
+ }
+ if (parent != null) {
+ builder.setParameter(Parameters.PARENT, parent);
+ }
+ if (routing != null) {
+ builder.setParameter(Parameters.ROUTING, routing);
+ }
+ if (version != null) {
+ builder.setParameter(Parameters.VERSION, version);
+ builder.setParameter(Parameters.VERSION_TYPE, "external");
+ }
+ bulkaction.addAction(builder.build());
+ break;
+ case UPDATE:
+ // doc: https://www.cnblogs.com/crystaltu/articles/6992935.html
+ // doc: https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html
+ Map updateDoc = new HashMap();
+ updateDoc.put("doc", data);
+ updateDoc.put("doc_as_upsert", true);
+ Update.Builder update = null;
+ if (this.enableWriteNull) {
+ // write: {a:"1",b:null}
+ update = new Update.Builder(
+ JSONObject.toJSONString(updateDoc, JSONWriter.Feature.WriteMapNullValue,
+ JSONWriter.Feature.WriteEnumUsingToString));
+ // 在DEFAULT_GENERATE_FEATURE基础上，只增加了SerializerFeature.WRITE_MAP_NULL_FEATURES
+ } else {
+ // write: {"a":"1"}
+ update = new Update.Builder(JSONObject.toJSONString(updateDoc));
+ }
+ if (id != null) {
+ update.id(id);
+ }
+ if (parent != null) {
+ update.setParameter(Parameters.PARENT, parent);
+ }
+ if (routing != null) {
+ update.setParameter(Parameters.ROUTING, routing);
+ }
+ // version type [EXTERNAL] is not supported by the update API
+ if (version != null) {
+ update.setParameter(Parameters.VERSION, version);
+ }
+ bulkaction.addAction(update.build());
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ if (dirtyDataNumber >= totalNumber) {
+ // all batch is dirty data
+ LOGGER.warn("all this batch is dirty data, dirtyDataNumber: {} totalDataNumber: {}", dirtyDataNumber,
+ totalNumber);
+ return;
+ }
+
+ BulkResult bulkResult = null;
+ try {
+ bulkResult = RetryUtil.executeWithRetry(new Callable() {
+ @Override
+ public BulkResult call() throws Exception {
+ JestResult jestResult = esClient.bulkInsert(bulkaction);
+ if (jestResult.isSucceeded()) {
+ return null;
+ }
+ String msg = String.format("response code: [%d] error :[%s]", jestResult.getResponseCode(),
+ jestResult.getErrorMessage());
+ LOGGER.warn(msg);
+ if (esClient.isBulkResult(jestResult)) {
+ BulkResult brst = (BulkResult) jestResult;
+ List failedItems = brst.getFailedItems();
+ for (BulkResult.BulkResultItem item : failedItems) {
+ if (item.status != 400) {
+ // 400 BAD_REQUEST 如果非数据异常,请求异常,则不允许忽略
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.ES_INDEX_INSERT,
+ String.format("status:[%d], error: %s", item.status, item.error));
+ } else {
+ // 如果用户选择不忽略解析错误,则抛异常,默认为忽略
+ if (!Key.isIgnoreParseError(conf)) {
+ throw new NoReRunException(ElasticSearchWriterErrorCode.ES_INDEX_INSERT,
+ String.format(
+ "status:[%d], error: %s, config not ignoreParseError so throw this error",
+ item.status, item.error));
+ }
+ }
+ }
+ return brst;
+ } else {
+ Integer status = esClient.getStatus(jestResult);
+ switch (status) {
+ case 429: // TOO_MANY_REQUESTS
+ LOGGER.warn("server response too many requests, so auto reduce speed");
+ break;
+ default:
+ break;
+ }
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.ES_INDEX_INSERT,
+ jestResult.getErrorMessage());
+ }
+ }
+ }, this.trySize, this.tryInterval, false, Arrays.asList(DataXException.class));
+ } catch (Exception e) {
+ if (Key.isIgnoreWriteError(this.conf)) {
+ LOGGER.warn(String.format("Retry [%d] write failed, ignore the error, continue to write!", trySize));
+ } else {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.ES_INDEX_INSERT, e.getMessage(), e);
+ }
+ }
+
+ if (null != bulkResult) {
+ List items = bulkResult.getItems();
+ for (int idx = 0; idx < items.size(); ++idx) {
+ BulkResult.BulkResultItem item = items.get(idx);
+ if (item.error != null && !"".equals(item.error)) {
+ super.getTaskPluginCollector().collectDirtyRecord(writerBuffer.get(idx),
+ String.format("status:[%d], error: %s", item.status, item.error));
+ }
+ }
+ }
+ }
+
+ private int getRecordColumnIndex(Record record, String columnName) {
+ if (colNameToIndexMap.containsKey(columnName)) {
+ return colNameToIndexMap.get(columnName);
+ }
+
+ List columns = new ArrayList();
+ int index = -1;
+ for (int i=0; i 1) {
+ throw DataXException.asDataXException(
+ ElasticSearchWriterErrorCode.RECORD_FIELD_NOT_FOUND,
+ "record has multiple columns found by name: " + columnName);
+ }
+
+ colNameToIndexMap.put(columnName, index);
+ return index;
+ }
+
+ private String processIDCombineFields(Record record, ElasticSearchColumn esColumn) {
+ List values = new ArrayList(esColumn.getCombineFields().size());
+ for (String field : esColumn.getCombineFields()) {
+ int colIndex = getRecordColumnIndex(record, field);
+ Column col = record.getColumnNumber() <= colIndex ? null : record.getColumn(colIndex);
+ if (col == null) {
+ throw DataXException.asDataXException(ElasticSearchWriterErrorCode.RECORD_FIELD_NOT_FOUND, field);
+ }
+ values.add(col.asString());
+ }
+ return Joiner.on(esColumn.getCombineFieldsValueSeparator()).join(values);
+ }
+
+ @Override
+ public void post() {
+ }
+
+ @Override
+ public void destroy() {
+ try {
+ this.esClient.closeJestClient();
+ } catch (Exception e) {
+ LOGGER.warn("ignore close jest client error: {}", e.getMessage());
+ }
+ }
+
+ }
+}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchWriterErrorCode.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchWriterErrorCode.java
new file mode 100644
index 00000000..c9b02532
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchWriterErrorCode.java
@@ -0,0 +1,41 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter;
+
+import com.alibaba.datax.common.spi.ErrorCode;
+
+public enum ElasticSearchWriterErrorCode implements ErrorCode {
+ BAD_CONFIG_VALUE("ESWriter-00", "The value you configured is not valid."),
+ ES_INDEX_DELETE("ESWriter-01", "Delete index error."),
+ ES_INDEX_CREATE("ESWriter-02", "Index creation error."),
+ ES_MAPPINGS("ESWriter-03", "The mappings error."),
+ ES_INDEX_INSERT("ESWriter-04", "Insert data error."),
+ ES_ALIAS_MODIFY("ESWriter-05", "Alias modification error."),
+ JSON_PARSE("ESWrite-06", "Json format parsing error"),
+ UPDATE_WITH_ID("ESWrite-07", "Update mode must specify column type with id"),
+ RECORD_FIELD_NOT_FOUND("ESWrite-08", "Field does not exist in the original table"),
+ ES_GET_SETTINGS("ESWriter-09", "get settings failed");
+ ;
+
+ private final String code;
+ private final String description;
+
+ ElasticSearchWriterErrorCode(String code, String description) {
+ this.code = code;
+ this.description = description;
+ }
+
+ @Override
+ public String getCode() {
+ return this.code;
+ }
+
+ @Override
+ public String getDescription() {
+ return this.description;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Code:[%s], Description:[%s]. ", this.code,
+ this.description);
+ }
+}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/JsonPathUtil.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/JsonPathUtil.java
new file mode 100644
index 00000000..e7619e7c
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/JsonPathUtil.java
@@ -0,0 +1,28 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter;
+
+import java.util.List;
+
+import com.alibaba.fastjson2.JSONObject;
+
+public class JsonPathUtil {
+
+ public static JSONObject getJsonObject(List paths, JSONObject data) {
+ if (null == paths || paths.isEmpty()) {
+ return data;
+ }
+
+ if (null == data) {
+ return null;
+ }
+
+ JSONObject dataTmp = data;
+ for (String each : paths) {
+ if (null != dataTmp) {
+ dataTmp = dataTmp.getJSONObject(each);
+ } else {
+ return null;
+ }
+ }
+ return dataTmp;
+ }
+}
\ No newline at end of file
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/JsonUtil.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/JsonUtil.java
new file mode 100644
index 00000000..ad6c01be
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/JsonUtil.java
@@ -0,0 +1,54 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter;
+
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONException;
+import com.alibaba.fastjson2.JSONObject;
+
+/**
+ * @author bozu
+ * @date 2021/01/06
+ */
+public class JsonUtil {
+
+ /**
+ * 合并两个json
+ * @param source 源json
+ * @param target 目标json
+ * @return 合并后的json
+ * @throws JSONException
+ */
+ public static String mergeJsonStr(String source, String target) throws JSONException {
+ if(source == null) {
+ return target;
+ }
+ if(target == null) {
+ return source;
+ }
+ return JSON.toJSONString(deepMerge(JSON.parseObject(source), JSON.parseObject(target)));
+ }
+
+ /**
+ * 深度合并两个json对象，将source的值，merge到target中
+ * @param source 源json
+ * @param target 目标json
+ * @return 合并后的json
+ * @throws JSONException
+ */
+ private static JSONObject deepMerge(JSONObject source, JSONObject target) throws JSONException {
+ for (String key: source.keySet()) {
+ Object value = source.get(key);
+ if (target.containsKey(key)) {
+ // existing value for "key" - recursively deep merge:
+ if (value instanceof JSONObject) {
+ JSONObject valueJson = (JSONObject)value;
+ deepMerge(valueJson, target.getJSONObject(key));
+ } else {
+ target.put(key, value);
+ }
+ } else {
+ target.put(key, value);
+ }
+ }
+ return target;
+ }
+}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/Key.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/Key.java
index 0f2d3f5c..fcaac935 100644
--- a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/Key.java
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/Key.java
@@ -1,9 +1,13 @@
package com.alibaba.datax.plugin.writer.elasticsearchwriter;
import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.TypeReference;
+
import org.apache.commons.lang3.StringUtils;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
public final class Key {
@@ -37,31 +41,35 @@ public final class Key {
public static String getEndpoint(Configuration conf) {
- return conf.getNecessaryValue("endpoint", ESWriterErrorCode.BAD_CONFIG_VALUE);
+ return conf.getNecessaryValue("endpoint", ElasticSearchWriterErrorCode.BAD_CONFIG_VALUE);
}
- public static String getAccessID(Configuration conf) {
- return conf.getString("accessId", "");
+ public static String getUsername(Configuration conf) {
+ return conf.getString("username", conf.getString("accessId"));
}
- public static String getAccessKey(Configuration conf) {
- return conf.getString("accessKey", "");
+ public static String getPassword(Configuration conf) {
+ return conf.getString("password", conf.getString("accessKey"));
}
public static int getBatchSize(Configuration conf) {
- return conf.getInt("batchSize", 1000);
+ return conf.getInt("batchSize", 1024);
}
public static int getTrySize(Configuration conf) {
return conf.getInt("trySize", 30);
}
+ public static long getTryInterval(Configuration conf) {
+ return conf.getLong("tryInterval", 60000L);
+ }
+
public static int getTimeout(Configuration conf) {
return conf.getInt("timeout", 600000);
}
- public static boolean isCleanup(Configuration conf) {
- return conf.getBool("cleanup", false);
+ public static boolean isTruncate(Configuration conf) {
+ return conf.getBool("truncate", conf.getBool("cleanup", false));
}
public static boolean isDiscovery(Configuration conf) {
@@ -69,7 +77,7 @@ public final class Key {
}
public static boolean isCompression(Configuration conf) {
- return conf.getBool("compression", true);
+ return conf.getBool("compress", conf.getBool("compression", true));
}
public static boolean isMultiThread(Configuration conf) {
@@ -77,9 +85,17 @@ public final class Key {
}
public static String getIndexName(Configuration conf) {
- return conf.getNecessaryValue("index", ESWriterErrorCode.BAD_CONFIG_VALUE);
+ return conf.getNecessaryValue("index", ElasticSearchWriterErrorCode.BAD_CONFIG_VALUE);
}
+ public static String getDeleteBy(Configuration conf) {
+ return conf.getString("deleteBy");
+ }
+
+
+ /**
+ * TODO: 在7.0开始，一个索引只能建一个Type为_doc
+ * */
public static String getTypeName(Configuration conf) {
String indexType = conf.getString("indexType");
if(StringUtils.isBlank(indexType)){
@@ -128,4 +144,58 @@ public final class Key {
public static boolean getDynamic(Configuration conf) {
return conf.getBool("dynamic", false);
}
+
+ public static String getDstDynamic(Configuration conf) {
+ return conf.getString("dstDynamic");
+ }
+
+ public static String getDiscoveryFilter(Configuration conf){
+ return conf.getString("discoveryFilter","_all");
+ }
+
+ public static Boolean getVersioning(Configuration conf) {
+ return conf.getBool("versioning", false);
+ }
+
+ public static Long getUnifiedVersion(Configuration conf) {
+ return conf.getLong("version", System.currentTimeMillis());
+ }
+
+ public static Map getUrlParams(Configuration conf) {
+ return conf.getMap("urlParams", new HashMap());
+ }
+
+ public static Integer getESVersion(Configuration conf) {
+ return conf.getInt("esVersion");
+ }
+
+ public static String getMasterTimeout(Configuration conf) {
+ return conf.getString("masterTimeout", "5m");
+ }
+
+ public static boolean isEnableNullUpdate(Configuration conf) {
+ return conf.getBool("enableWriteNull", true);
+ }
+
+ public static String getFieldDelimiter(Configuration conf) {
+ return conf.getString("fieldDelimiter", "");
+ }
+
+ public static PrimaryKeyInfo getPrimaryKeyInfo(Configuration conf) {
+ String primaryKeyInfoString = conf.getString("primaryKeyInfo");
+ if (StringUtils.isNotBlank(primaryKeyInfoString)) {
+ return JSON.parseObject(primaryKeyInfoString, new TypeReference() {});
+ } else {
+ return null;
+ }
+ }
+
+ public static List getEsPartitionColumn(Configuration conf) {
+ String esPartitionColumnString = conf.getString("esPartitionColumn");
+ if (StringUtils.isNotBlank(esPartitionColumnString)) {
+ return JSON.parseObject(esPartitionColumnString, new TypeReference>() {});
+ } else {
+ return null;
+ }
+ }
}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/NoReRunException.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/NoReRunException.java
new file mode 100644
index 00000000..52064e58
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/NoReRunException.java
@@ -0,0 +1,16 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.spi.ErrorCode;
+
+public class NoReRunException extends DataXException {
+ public NoReRunException(String errorMessage) {
+ super(errorMessage);
+ }
+
+ public NoReRunException(ErrorCode errorCode, String errorMessage) {
+ super(errorCode, errorMessage);
+ }
+
+ private static final long serialVersionUID = 1L;
+}
\ No newline at end of file
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/PartitionColumn.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/PartitionColumn.java
new file mode 100644
index 00000000..b99829b2
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/PartitionColumn.java
@@ -0,0 +1,42 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter;
+
+public class PartitionColumn {
+ private String name;
+ // like: DATA
+ private String metaType;
+ private String comment;
+ // like: VARCHAR
+ private String type;
+
+ public String getName() {
+ return name;
+ }
+
+ public String getMetaType() {
+ return metaType;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void setMetaType(String metaType) {
+ this.metaType = metaType;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+}
\ No newline at end of file
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/PrimaryKeyInfo.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/PrimaryKeyInfo.java
new file mode 100644
index 00000000..b5821f51
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/PrimaryKeyInfo.java
@@ -0,0 +1,47 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter;
+
+import java.util.List;
+
+public class PrimaryKeyInfo {
+
+ /**
+ * 主键类型:PrimaryKeyTypeEnum
+ *
+ * pk: 单个(业务)主键 specific: 联合主键
+ */
+ private String type;
+
+ /**
+ * 用户定义的联合主键的连接符号
+ */
+ private String fieldDelimiter;
+
+ /**
+ * 主键的列的名称
+ */
+ private List column;
+
+ public String getType() {
+ return type;
+ }
+
+ public String getFieldDelimiter() {
+ return fieldDelimiter;
+ }
+
+ public List getColumn() {
+ return column;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public void setFieldDelimiter(String fieldDelimiter) {
+ this.fieldDelimiter = fieldDelimiter;
+ }
+
+ public void setColumn(List column) {
+ this.column = column;
+ }
+}
\ No newline at end of file
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/ClusterInfo.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/ClusterInfo.java
new file mode 100644
index 00000000..173bc9e2
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/ClusterInfo.java
@@ -0,0 +1,35 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter.jest;
+
+import com.google.gson.Gson;
+import io.searchbox.action.AbstractAction;
+import io.searchbox.client.config.ElasticsearchVersion;
+
+public class ClusterInfo extends AbstractAction {
+ @Override
+ protected String buildURI(ElasticsearchVersion elasticsearchVersion) {
+ return "";
+ }
+
+ @Override
+ public String getRestMethodName() {
+ return "GET";
+ }
+
+ @Override
+ public ClusterInfoResult createNewElasticSearchResult(String responseBody, int statusCode, String reasonPhrase, Gson gson) {
+ return createNewElasticSearchResult(new ClusterInfoResult(gson), responseBody, statusCode, reasonPhrase, gson);
+ }
+
+ public static class Builder extends AbstractAction.Builder {
+
+ public Builder() {
+ setHeader("accept", "application/json");
+ setHeader("content-type", "application/json");
+ }
+
+ @Override
+ public ClusterInfo build() {
+ return new ClusterInfo();
+ }
+ }
+}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/ClusterInfoResult.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/ClusterInfoResult.java
new file mode 100644
index 00000000..b4f49a37
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/ClusterInfoResult.java
@@ -0,0 +1,49 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter.jest;
+
+import com.google.gson.Gson;
+import io.searchbox.client.JestResult;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ClusterInfoResult extends JestResult {
+
+ private static final Pattern FIRST_NUMBER = Pattern.compile("\\d");
+
+ private static final int SEVEN = 7;
+
+ public ClusterInfoResult(Gson gson) {
+ super(gson);
+ }
+
+ public ClusterInfoResult(JestResult source) {
+ super(source);
+ }
+
+ /**
+ * 判断es集群的部署版本是否大于7.x
+ * 大于7.x的es对于Index的type有较大改动，需要做额外判定
+ * 对于7.x与6.x版本的es都做过测试，返回符合预期;5.x以下版本直接try-catch后返回false，向下兼容
+ * @return
+ */
+ public Boolean isGreaterOrEqualThan7() throws Exception {
+ // 如果是没有权限，直接返回false，兼容老版本
+ if (responseCode == 403) {
+ return false;
+ }
+ if (!isSucceeded) {
+ throw new Exception(getJsonString());
+ }
+ try {
+ String version = jsonObject.getAsJsonObject("version").get("number").toString();
+ Matcher matcher = FIRST_NUMBER.matcher(version);
+ matcher.find();
+ String number = matcher.group();
+ Integer versionNum = Integer.valueOf(number);
+ return versionNum >= SEVEN;
+ } catch (Exception e) {
+ //5.x 以下版本不做兼容测试，如果返回json格式解析失败，有可能是以下版本，所以认为不大于7.x
+ return false;
+ }
+ }
+}
diff --git a/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/PutMapping7.java b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/PutMapping7.java
new file mode 100644
index 00000000..c9f1d6be
--- /dev/null
+++ b/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/PutMapping7.java
@@ -0,0 +1,39 @@
+package com.alibaba.datax.plugin.writer.elasticsearchwriter.jest;
+
+import io.searchbox.action.GenericResultAbstractAction;
+import io.searchbox.client.config.ElasticsearchVersion;
+
+public class PutMapping7 extends GenericResultAbstractAction {
+ protected PutMapping7(PutMapping7.Builder builder) {
+ super(builder);
+
+ this.indexName = builder.index;
+ this.payload = builder.source;
+ }
+
+ @Override
+ protected String buildURI(ElasticsearchVersion elasticsearchVersion) {
+ return super.buildURI(elasticsearchVersion) + "/_mapping";
+ }
+
+ @Override
+ public String getRestMethodName() {
+ return "PUT";
+ }
+
+ public static class Builder extends GenericResultAbstractAction.Builder {
+ private String index;
+ private Object source;
+
+ public Builder(String index, Object source) {
+ this.index = index;
+ this.source = source;
+ }
+
+ @Override
+ public PutMapping7 build() {
+ return new PutMapping7(this);
+ }
+ }
+
+}
diff --git a/elasticsearchwriter/src/main/resources/plugin.json b/elasticsearchwriter/src/main/resources/plugin.json
index b6e6384b..b39f1222 100644
--- a/elasticsearchwriter/src/main/resources/plugin.json
+++ b/elasticsearchwriter/src/main/resources/plugin.json
@@ -1,6 +1,6 @@
{
"name": "elasticsearchwriter",
- "class": "com.alibaba.datax.plugin.writer.elasticsearchwriter.ESWriter",
+ "class": "com.alibaba.datax.plugin.writer.elasticsearchwriter.ElasticSearchWriter",
"description": "适用于: 生产环境. 原理: TODO",
"developer": "alibaba"
}
\ No newline at end of file
diff --git a/ftpwriter/doc/ftpwriter.md b/ftpwriter/doc/ftpwriter.md
index 6b1b2687..a38a1052 100644
--- a/ftpwriter/doc/ftpwriter.md
+++ b/ftpwriter/doc/ftpwriter.md
@@ -24,7 +24,7 @@ FtpWriter实现了从DataX协议转为FTP文件功能，FTP文件本身是无结
我们不能做到：
-1. 单个文件不能支持并发写入。
+1. 单个文件并发写入。
## 3 功能说明
diff --git a/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/SftpHelperImpl.java b/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/SftpHelperImpl.java
index e6d78629..e748f12c 100644
--- a/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/SftpHelperImpl.java
+++ b/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/SftpHelperImpl.java
@@ -14,8 +14,8 @@ import org.slf4j.LoggerFactory;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.plugin.writer.ftpwriter.FtpWriterErrorCode;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONWriter;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
@@ -251,7 +251,7 @@ public class SftpHelperImpl implements IFtpHelper {
@SuppressWarnings("rawtypes")
Vector allFiles = this.channelSftp.ls(dir);
LOG.debug(String.format("ls: %s", JSON.toJSONString(allFiles,
- SerializerFeature.UseSingleQuotes)));
+ JSONWriter.Feature.UseSingleQuotes)));
for (int i = 0; i < allFiles.size(); i++) {
LsEntry le = (LsEntry) allFiles.get(i);
String strName = le.getFilename();
diff --git a/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/StandardFtpHelperImpl.java b/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/StandardFtpHelperImpl.java
index 8999b0a8..d5b9a746 100644
--- a/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/StandardFtpHelperImpl.java
+++ b/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/StandardFtpHelperImpl.java
@@ -18,8 +18,8 @@ import org.slf4j.LoggerFactory;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.plugin.writer.ftpwriter.FtpWriterErrorCode;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONWriter;
public class StandardFtpHelperImpl implements IFtpHelper {
private static final Logger LOG = LoggerFactory
@@ -244,7 +244,7 @@ public class StandardFtpHelperImpl implements IFtpHelper {
FTPFile[] fs = this.ftpClient.listFiles(dir);
// LOG.debug(JSON.toJSONString(this.ftpClient.listNames(dir)));
LOG.debug(String.format("ls: %s",
- JSON.toJSONString(fs, SerializerFeature.UseSingleQuotes)));
+ JSON.toJSONString(fs, JSONWriter.Feature.UseSingleQuotes)));
for (FTPFile ff : fs) {
String strName = ff.getName();
if (strName.startsWith(prefixFileName)) {
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/DefaultGdbMapper.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/DefaultGdbMapper.java
index 73a94cf5..2c015879 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/DefaultGdbMapper.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/DefaultGdbMapper.java
@@ -19,8 +19,8 @@ import com.alibaba.datax.plugin.writer.gdbwriter.Key;
import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbEdge;
import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbElement;
import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbVertex;
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson2.JSONArray;
+import com.alibaba.fastjson2.JSONObject;
import lombok.extern.slf4j.Slf4j;
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/ConfigHelper.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/ConfigHelper.java
index 178b5e7c..644f8898 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/ConfigHelper.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/ConfigHelper.java
@@ -12,8 +12,8 @@ import org.apache.commons.lang3.StringUtils;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.writer.gdbwriter.GdbWriterErrorCode;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONObject;
/**
* @author jerrywang
diff --git a/hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Hbase094xHelper.java b/hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Hbase094xHelper.java
index c3e2a212..b9f16b17 100644
--- a/hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Hbase094xHelper.java
+++ b/hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Hbase094xHelper.java
@@ -2,8 +2,8 @@ package com.alibaba.datax.plugin.reader.hbase094xreader;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.TypeReference;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.TypeReference;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.hadoop.fs.Path;
diff --git a/hbase094xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase094xwriter/Hbase094xHelper.java b/hbase094xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase094xwriter/Hbase094xHelper.java
index f671d31d..00b128f3 100644
--- a/hbase094xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase094xwriter/Hbase094xHelper.java
+++ b/hbase094xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase094xwriter/Hbase094xHelper.java
@@ -2,8 +2,8 @@ package com.alibaba.datax.plugin.writer.hbase094xwriter;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.TypeReference;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.TypeReference;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.hadoop.fs.Path;
diff --git a/hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/Hbase11xHelper.java b/hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/Hbase11xHelper.java
index 643072a9..82ad7122 100644
--- a/hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/Hbase11xHelper.java
+++ b/hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/Hbase11xHelper.java
@@ -2,8 +2,8 @@ package com.alibaba.datax.plugin.reader.hbase11xreader;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.TypeReference;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.TypeReference;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.hadoop.hbase.HBaseConfiguration;
diff --git a/hbase11xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java b/hbase11xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java
index 5309d1d9..71665a6b 100644
--- a/hbase11xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java
+++ b/hbase11xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java
@@ -2,8 +2,8 @@ package com.alibaba.datax.plugin.reader.hbase11xsqlreader;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.TypeReference;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.TypeReference;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.mapreduce.InputSplit;
diff --git a/hbase11xsqlreader/src/main/resources/plugin.json b/hbase11xsqlreader/src/main/resources/plugin.json
index e245ca27..162f5712 100644
--- a/hbase11xsqlreader/src/main/resources/plugin.json
+++ b/hbase11xsqlreader/src/main/resources/plugin.json
@@ -2,6 +2,6 @@
"name": "hbase11xsqlreader",
"class": "com.alibaba.datax.plugin.reader.hbase11xsqlreader.HbaseSQLReader",
"description": "useScene: prod. mechanism: Scan to read data.",
- "developer": "liwei.li, bug reported to : liwei.li@alibaba-inc.com"
+ "developer": "alibaba"
}
diff --git a/hbase11xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xsqlwriter/HbaseSQLHelper.java b/hbase11xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xsqlwriter/HbaseSQLHelper.java
index 41e57d4e..d1b23fdf 100644
--- a/hbase11xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xsqlwriter/HbaseSQLHelper.java
+++ b/hbase11xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xsqlwriter/HbaseSQLHelper.java
@@ -2,8 +2,8 @@ package com.alibaba.datax.plugin.writer.hbase11xsqlwriter;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.TypeReference;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.TypeReference;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.util.Pair;
diff --git a/hbase11xwriter/doc/hbase11xwriter.md b/hbase11xwriter/doc/hbase11xwriter.md
index ff20abe9..969f2e47 100644
--- a/hbase11xwriter/doc/hbase11xwriter.md
+++ b/hbase11xwriter/doc/hbase11xwriter.md
@@ -203,19 +203,20 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba
* 描述：要写入的hbase字段。index：指定该列对应reader端column的索引，从0开始；name：指定hbase表中的列，必须为 列族:列名 的格式；type：指定写入数据类型，用于转换HBase byte[]。配置格式如下：
```
-"column": [
- {
- "index":1,
- "name": "cf1:q1",
- "type": "string"
- },
- {
- "index":2,
- "name": "cf1:q2",
- "type": "string"
- }
- ］
-
+
+ "column": [
+ {
+ "index":1,
+ "name": "cf1:q1",
+ "type": "string"
+ },
+ {
+ "index":2,
+ "name": "cf1:q2",
+ "type": "string"
+ }
+ ］
+
```
* 必选：是
@@ -227,17 +228,17 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba
* 描述：要写入的hbase的rowkey列。index：指定该列对应reader端column的索引，从0开始，若为常量index为－1；type：指定写入数据类型，用于转换HBase byte[]；value：配置常量，常作为多个字段的拼接符。hbasewriter会将rowkeyColumn中所有列按照配置顺序进行拼接作为写入hbase的rowkey，不能全为常量。配置格式如下：
```
-"rowkeyColumn": [
- {
- "index":0,
- "type":"string"
- },
- {
- "index":-1,
- "type":"string",
- "value":"_"
- }
- ]
+ "rowkeyColumn": [
+ {
+ "index":0,
+ "type":"string"
+ },
+ {
+ "index":-1,
+ "type":"string",
+ "value":"_"
+ }
+ ]
```
@@ -250,19 +251,19 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba
* 描述：指定写入hbase的时间戳。支持：当前时间、指定时间列，指定时间，三者选一。若不配置表示用当前时间。index：指定对应reader端column的索引，从0开始，需保证能转换为long,若是Date类型，会尝试用yyyy-MM-dd HH:mm:ss和yyyy-MM-dd HH:mm:ss SSS去解析；若为指定时间index为－1；value：指定时间的值,long值。配置格式如下：
```
-"versionColumn":{
- "index":1
-}
+ "versionColumn":{
+ "index":1
+ }
```
或者
```
-"versionColumn":{
- "index":－1,
- "value":123456789
-}
+ "versionColumn":{
+ "index":－1,
+ "value":123456789
+ }
```
diff --git a/hbase11xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xwriter/Hbase11xHelper.java b/hbase11xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xwriter/Hbase11xHelper.java
index 94b13b60..2889b647 100644
--- a/hbase11xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xwriter/Hbase11xHelper.java
+++ b/hbase11xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xwriter/Hbase11xHelper.java
@@ -2,8 +2,8 @@ package com.alibaba.datax.plugin.writer.hbase11xwriter;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.TypeReference;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.TypeReference;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.hadoop.hbase.HBaseConfiguration;
diff --git a/hbase20xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase20xsqlreader/HBase20SQLReaderHelper.java b/hbase20xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase20xsqlreader/HBase20SQLReaderHelper.java
index 0edc993f..11bbf734 100644
--- a/hbase20xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase20xsqlreader/HBase20SQLReaderHelper.java
+++ b/hbase20xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase20xsqlreader/HBase20SQLReaderHelper.java
@@ -175,7 +175,7 @@ public class HBase20SQLReaderHelper {
if (querySql == null || querySql.isEmpty()) {
// 如果splitPoints为空，则根据splitKey自动切分，不过这种切分方式无法保证数据均分，且只支持整形和字符型列
if (splitPoints == null || splitPoints.isEmpty()) {
- LOG.info("Split accoring min and max value of splitColumn...");
+ LOG.info("Split according min and max value of splitColumn...");
Pair minMaxPK = getPkRange(configuration);
if (null == minMaxPK) {
throw DataXException.asDataXException(HBase20xSQLReaderErrorCode.ILLEGAL_SPLIT_PK,
@@ -208,7 +208,7 @@ public class HBase20SQLReaderHelper {
}
} else {
- LOG.info("Split accoring splitPoints...");
+ LOG.info("Split according splitPoints...");
// 根据指定splitPoints进行切分
rangeList = buildSplitRange();
}
diff --git a/hbase20xsqlreader/src/main/resources/plugin.json b/hbase20xsqlreader/src/main/resources/plugin.json
index 45856411..4a7b4edf 100644
--- a/hbase20xsqlreader/src/main/resources/plugin.json
+++ b/hbase20xsqlreader/src/main/resources/plugin.json
@@ -2,6 +2,6 @@
"name": "hbase20xsqlreader",
"class": "com.alibaba.datax.plugin.reader.hbase20xsqlreader.HBase20xSQLReader",
"description": "useScene: prod. mechanism: read data from phoenix through queryserver.",
- "developer": "bake"
+ "developer": "alibaba"
}
diff --git a/hbase20xsqlwriter/src/main/resources/plugin.json b/hbase20xsqlwriter/src/main/resources/plugin.json
index 91b7069f..93d3002a 100755
--- a/hbase20xsqlwriter/src/main/resources/plugin.json
+++ b/hbase20xsqlwriter/src/main/resources/plugin.json
@@ -2,6 +2,6 @@
"name": "hbase20xsqlwriter",
"class": "com.alibaba.datax.plugin.writer.hbase20xsqlwriter.HBase20xSQLWriter",
"description": "useScene: prod. mechanism: use hbase sql UPSERT to put data, index tables will be updated too.",
- "developer": "bake"
+ "developer": "alibaba"
}
diff --git a/hdfsreader/pom.xml b/hdfsreader/pom.xml
index 5d07dc25..a5c2da2c 100644
--- a/hdfsreader/pom.xml
+++ b/hdfsreader/pom.xml
@@ -16,6 +16,17 @@
2.7.1
+
+ 
+用户咨询支持：
-- DataX开源用户交流群2
- - 
+钉钉群目前暂时受到了一些管控策略影响，建议大家有问题优先在这里提交问题 Issue，DataX研发和社区会定期回答Issue中的问题，知识库丰富后也能帮助到后来的使用者。
-- DataX开源用户交流群3
- - 
-- DataX开源用户交流群4
- - 
-
-- DataX开源用户交流群5
- - 
-
-- DataX开源用户交流群6
- - 
diff --git a/adbmysqlwriter/doc/adbmysqlwriter.md b/adbmysqlwriter/doc/adbmysqlwriter.md
new file mode 100644
index 00000000..27ac6b10
--- /dev/null
+++ b/adbmysqlwriter/doc/adbmysqlwriter.md
@@ -0,0 +1,338 @@
+# DataX AdbMysqlWriter
+
+
+---
+
+
+## 1 快速介绍
+
+AdbMysqlWriter 插件实现了写入数据到 ADB MySQL 目的表的功能。在底层实现上， AdbMysqlWriter 通过 JDBC 连接远程 ADB MySQL 数据库，并执行相应的 `insert into ...` 或者 ( `replace into ...` ) 的 SQL 语句将数据写入 ADB MySQL，内部会分批次提交入库。
+
+AdbMysqlWriter 面向ETL开发工程师，他们使用 AdbMysqlWriter 从数仓导入数据到 ADB MySQL。同时 AdbMysqlWriter 亦可以作为数据迁移工具为DBA等用户提供服务。
+
+
+## 2 实现原理
+
+AdbMysqlWriter 通过 DataX 框架获取 Reader 生成的协议数据，AdbMysqlWriter 通过 JDBC 连接远程 ADB MySQL 数据库，并执行相应的 `insert into ...` 或者 ( `replace into ...` ) 的 SQL 语句将数据写入 ADB MySQL。
+
+
+* `insert into...`(遇到主键重复时会自动忽略当前写入数据，不做更新，作用等同于`insert ignore into`)
+
+##### 或者
+
+* `replace into...`(没有遇到主键/唯一性索引冲突时，与 insert into 行为一致，冲突时会用新行替换原有行所有字段) 的语句写入数据到 MySQL。出于性能考虑，采用了 `PreparedStatement + Batch`，并且设置了：`rewriteBatchedStatements=true`，将数据缓冲到线程上下文 Buffer 中，当 Buffer 累计到预定阈值时，才发起写入请求。
+
+