diff --git a/core/src/main/conf/.secret.properties b/core/src/main/conf/.secret.properties index b807f8ad..45b4b127 100755 --- a/core/src/main/conf/.secret.properties +++ b/core/src/main/conf/.secret.properties @@ -6,4 +6,4 @@ current.publicKey= current.privateKey= current.service.username= current.service.password= - +dp.public.k= diff --git a/core/src/main/java/com/alibaba/datax/core/Engine.java b/core/src/main/java/com/alibaba/datax/core/Engine.java index 4ba9fc18..6e7185a7 100755 --- a/core/src/main/java/com/alibaba/datax/core/Engine.java +++ b/core/src/main/java/com/alibaba/datax/core/Engine.java @@ -9,10 +9,7 @@ import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.common.util.MessageSource; import com.alibaba.datax.core.job.JobContainer; import com.alibaba.datax.core.taskgroup.TaskGroupContainer; -import com.alibaba.datax.core.util.ConfigParser; -import com.alibaba.datax.core.util.ConfigurationValidate; -import com.alibaba.datax.core.util.ExceptionTracker; -import com.alibaba.datax.core.util.FrameworkErrorCode; +import com.alibaba.datax.core.util.*; import com.alibaba.datax.core.util.container.CoreConstant; import com.alibaba.datax.core.util.container.LoadUtil; import org.apache.commons.cli.BasicParser; @@ -50,11 +47,12 @@ public class Engine { boolean isJob = !("taskGroup".equalsIgnoreCase(allConf .getString(CoreConstant.DATAX_CORE_CONTAINER_MODEL))); //JobContainer会在schedule后再行进行设置和调整值 - int channelNumber =0; + int channelNumber = 0; AbstractContainer container; long instanceId; int taskGroupId = -1; if (isJob) { + JobPwdDescryptUtil.decrypt(allConf); allConf.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_MODE, RUNTIME_MODE); container = new JobContainer(allConf); instanceId = allConf.getLong( @@ -75,14 +73,14 @@ public class Engine { boolean perfReportEnable = allConf.getBool(CoreConstant.DATAX_CORE_REPORT_DATAX_PERFLOG, true); //standalone模式的 datax shell任务不进行汇报 - if(instanceId == -1){ + if (instanceId == -1) { perfReportEnable = false; } Configuration jobInfoConfig = allConf.getConfiguration(CoreConstant.DATAX_JOB_JOBINFO); //初始化PerfTrace PerfTrace perfTrace = PerfTrace.getInstance(isJob, instanceId, taskGroupId, traceEnable); - perfTrace.setJobInfo(jobInfoConfig,perfReportEnable,channelNumber); + perfTrace.setJobInfo(jobInfoConfig, perfReportEnable, channelNumber); container.start(); } @@ -96,12 +94,12 @@ public class Engine { filterSensitiveConfiguration(jobContent); - jobConfWithSetting.set("content",jobContent); + jobConfWithSetting.set("content", jobContent); return jobConfWithSetting.beautify(); } - public static Configuration filterSensitiveConfiguration(Configuration configuration){ + public static Configuration filterSensitiveConfiguration(Configuration configuration) { Set keys = configuration.getKeys(); for (final String key : keys) { boolean isSensitive = StringUtils.endsWithIgnoreCase(key, "password") @@ -171,8 +169,8 @@ public class Engine { /** * -1 表示未能解析到 jobId - * - * only for dsc & ds & datax 3 update + *

+ * only for dsc & ds & datax 3 update */ private static long parseJobIdFromUrl(List patternStringList, String url) { long result = -1; diff --git a/core/src/main/java/com/alibaba/datax/core/util/JobPwdDescryptUtil.java b/core/src/main/java/com/alibaba/datax/core/util/JobPwdDescryptUtil.java new file mode 100644 index 00000000..e2bdc4ca --- /dev/null +++ b/core/src/main/java/com/alibaba/datax/core/util/JobPwdDescryptUtil.java @@ -0,0 +1,31 @@ +package com.alibaba.datax.core.util; + +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.core.util.container.CoreConstant; +import org.apache.commons.lang.StringUtils; + +/**job数据库密码解密 + * @Author weizhao.dong + * @Date 2023/3/23 14:37 + * @Version 1.0 + */ +public class JobPwdDescryptUtil { + + public static void decrypt(Configuration configuration) { + if (configuration.getBool(CoreConstant.DATAX_JOB_SETTING_PASSWD_ENCRYPT, false)) { + String readerPwd = configuration.getString(CoreConstant.DATA_JOB_READER_PARAMETER_PASSWORD); + String writePwd = configuration.getString(CoreConstant.DATA_JOB_WRITER_PARAMETER_PASSWORD); + //加密key + String key = SecretUtil.getSecurityProperties().getProperty(CoreConstant.PASSWD_KEY); + if (StringUtils.isEmpty(key)) { + key = CoreConstant.PASSWD_KEY_DEFAULT; + } + if (StringUtils.isNotEmpty(readerPwd)) { + configuration.set(CoreConstant.DATA_JOB_READER_PARAMETER_PASSWORD, SecretUtil.decrypt3DES(readerPwd, key)); + } + if (StringUtils.isNotEmpty(writePwd)) { + configuration.set(CoreConstant.DATA_JOB_WRITER_PARAMETER_PASSWORD, SecretUtil.decrypt3DES(writePwd, key)); + } + } + } +} diff --git a/core/src/main/java/com/alibaba/datax/core/util/SecretUtil.java b/core/src/main/java/com/alibaba/datax/core/util/SecretUtil.java index 1a576aaa..e19cfbab 100755 --- a/core/src/main/java/com/alibaba/datax/core/util/SecretUtil.java +++ b/core/src/main/java/com/alibaba/datax/core/util/SecretUtil.java @@ -437,4 +437,13 @@ public class SecretUtil { } return versionKeyMap; } + + public static void main(String[] args) { + String key="1qaz2wsx"; + String passwd="BrPN#dEzqm"; + String encrypt= SecretUtil.encrypt3DES(passwd,key); + System.out.println("encrypt = " + encrypt); + System.out.println(SecretUtil.decrypt3DES(encrypt,key)); + + } } diff --git a/core/src/main/java/com/alibaba/datax/core/util/container/CoreConstant.java b/core/src/main/java/com/alibaba/datax/core/util/container/CoreConstant.java index 6a0b6205..e81e195c 100755 --- a/core/src/main/java/com/alibaba/datax/core/util/container/CoreConstant.java +++ b/core/src/main/java/com/alibaba/datax/core/util/container/CoreConstant.java @@ -97,6 +97,8 @@ public class CoreConstant { public static final String DATAX_JOB_SETTING_DRYRUN = "job.setting.dryRun"; + public static final String DATAX_JOB_SETTING_PASSWD_ENCRYPT = "job.setting.passwdEncrypt"; + public static final String DATAX_JOB_PREHANDLER_PLUGINTYPE = "job.preHandler.pluginType"; public static final String DATAX_JOB_PREHANDLER_PLUGINNAME = "job.preHandler.pluginName"; @@ -104,6 +106,13 @@ public class CoreConstant { public static final String DATAX_JOB_POSTHANDLER_PLUGINTYPE = "job.postHandler.pluginType"; public static final String DATAX_JOB_POSTHANDLER_PLUGINNAME = "job.postHandler.pluginName"; + + public static final String DATA_JOB_READER_PARAMETER_PASSWORD="job.content[0].reader.parameter.password"; + + public static final String DATA_JOB_WRITER_PARAMETER_PASSWORD="job.content[0].writer.parameter.password"; + + + // ----------------------------- 局部使用的变量 public static final String JOB_WRITER = "reader"; @@ -148,6 +157,8 @@ public class CoreConstant { public static final String CURRENT_SERVICE_USERNAME = "current.service.username"; public static final String CURRENT_SERVICE_PASSWORD = "current.service.password"; + public static final String PASSWD_KEY="dp.public.k"; + public static final String PASSWD_KEY_DEFAULT="dwz1qaz2wsx"; // ----------------------------- 环境变量 --------------------------------- diff --git a/core/src/test/java/ConfigurationTest.java b/core/src/test/java/ConfigurationTest.java new file mode 100644 index 00000000..5bb3b4cc --- /dev/null +++ b/core/src/test/java/ConfigurationTest.java @@ -0,0 +1,18 @@ +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.fastjson2.JSON; +import org.junit.Test; + +import java.io.File; + +/** + * @Author weizhao.dong + * @Date 2023/3/22 18:26 + * @Version 1.0 + */ +public class ConfigurationTest { + @Test + public void configParseTest(){ + Configuration configuration=Configuration.from(new File("/Users/weizhao.dong/Documents/soft/datax_install_d/script/dwd_g2park_inout_report_s.json")); + System.out.println(JSON.toJSONString(configuration)); + } +}