github/DataX
5
0
Fork 0
mirror of https://github.com/alibaba/DataX.git synced 2025-05-02 07:59:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加job数据库密码加密功能

Browse Source
This commit is contained in:
weizhao1 2023-03-23 16:04:41 +08:00
parent 2f5c9cf91e
commit e51eb2e996
6 changed files with 79 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/main/conf/.secret.properties
View File

@ -6,4 +6,4 @@ current.publicKey=
current.privateKey=
current.service.username=
current.service.password=
dp.public.k=

20
core/src/main/java/com/alibaba/datax/core/Engine.java
View File

@ -9,10 +9,7 @@ import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.common.util.MessageSource;
import com.alibaba.datax.core.job.JobContainer;
import com.alibaba.datax.core.taskgroup.TaskGroupContainer;
import com.alibaba.datax.core.util.ConfigParser;
import com.alibaba.datax.core.util.ConfigurationValidate;
import com.alibaba.datax.core.util.ExceptionTracker;
import com.alibaba.datax.core.util.FrameworkErrorCode;
import com.alibaba.datax.core.util.*;
import com.alibaba.datax.core.util.container.CoreConstant;
import com.alibaba.datax.core.util.container.LoadUtil;
import org.apache.commons.cli.BasicParser;
@ -50,11 +47,12 @@ public class Engine {
boolean isJob = !("taskGroup".equalsIgnoreCase(allConf
.getString(CoreConstant.DATAX_CORE_CONTAINER_MODEL)));
//JobContainer会在schedule后再行进行设置和调整值
int channelNumber =0;
int channelNumber = 0;
AbstractContainer container;
long instanceId;
int taskGroupId = -1;
if (isJob) {
JobPwdDescryptUtil.decrypt(allConf);
allConf.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_MODE, RUNTIME_MODE);
container = new JobContainer(allConf);
instanceId = allConf.getLong(
@ -75,14 +73,14 @@ public class Engine {
boolean perfReportEnable = allConf.getBool(CoreConstant.DATAX_CORE_REPORT_DATAX_PERFLOG, true);
//standalone模式的 datax shell任务不进行汇报
if(instanceId == -1){
if (instanceId == -1) {
perfReportEnable = false;
}
Configuration jobInfoConfig = allConf.getConfiguration(CoreConstant.DATAX_JOB_JOBINFO);
//初始化PerfTrace
PerfTrace perfTrace = PerfTrace.getInstance(isJob, instanceId, taskGroupId, traceEnable);
perfTrace.setJobInfo(jobInfoConfig,perfReportEnable,channelNumber);
perfTrace.setJobInfo(jobInfoConfig, perfReportEnable, channelNumber);
container.start();
}
@ -96,12 +94,12 @@ public class Engine {
filterSensitiveConfiguration(jobContent);
jobConfWithSetting.set("content",jobContent);
jobConfWithSetting.set("content", jobContent);
return jobConfWithSetting.beautify();
}
public static Configuration filterSensitiveConfiguration(Configuration configuration){
public static Configuration filterSensitiveConfiguration(Configuration configuration) {
Set<String> keys = configuration.getKeys();
for (final String key : keys) {
boolean isSensitive = StringUtils.endsWithIgnoreCase(key, "password")
@ -171,8 +169,8 @@ public class Engine {
/**
* -1 表示未能解析到 jobId
*
* only for dsc & ds & datax 3 update
* <p>
* only for dsc & ds & datax 3 update
*/
private static long parseJobIdFromUrl(List<String> patternStringList, String url) {
long result = -1;

31
core/src/main/java/com/alibaba/datax/core/util/JobPwdDescryptUtil.java Normal file
View File

@ -0,0 +1,31 @@
package com.alibaba.datax.core.util;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.core.util.container.CoreConstant;
import org.apache.commons.lang.StringUtils;
/**job数据库密码解密
* @Author weizhao.dong
* @Date 2023/3/23 14:37
* @Version 1.0
*/
public class JobPwdDescryptUtil {
public static void decrypt(Configuration configuration) {
if (configuration.getBool(CoreConstant.DATAX_JOB_SETTING_PASSWD_ENCRYPT, false)) {
String readerPwd = configuration.getString(CoreConstant.DATA_JOB_READER_PARAMETER_PASSWORD);
String writePwd = configuration.getString(CoreConstant.DATA_JOB_WRITER_PARAMETER_PASSWORD);
//加密key
String key = SecretUtil.getSecurityProperties().getProperty(CoreConstant.PASSWD_KEY);
if (StringUtils.isEmpty(key)) {
key = CoreConstant.PASSWD_KEY_DEFAULT;
}
if (StringUtils.isNotEmpty(readerPwd)) {
configuration.set(CoreConstant.DATA_JOB_READER_PARAMETER_PASSWORD, SecretUtil.decrypt3DES(readerPwd, key));
}
if (StringUtils.isNotEmpty(writePwd)) {
configuration.set(CoreConstant.DATA_JOB_WRITER_PARAMETER_PASSWORD, SecretUtil.decrypt3DES(writePwd, key));
}
}
}
}

9
core/src/main/java/com/alibaba/datax/core/util/SecretUtil.java
View File

@ -437,4 +437,13 @@ public class SecretUtil {
}
return versionKeyMap;
}
public static void main(String[] args) {
String key="1qaz2wsx";
String passwd="BrPN#dEzqm";
String encrypt= SecretUtil.encrypt3DES(passwd,key);
System.out.println("encrypt = " + encrypt);
System.out.println(SecretUtil.decrypt3DES(encrypt,key));
}
}

11
core/src/main/java/com/alibaba/datax/core/util/container/CoreConstant.java
View File

@ -97,6 +97,8 @@ public class CoreConstant {
public static final String DATAX_JOB_SETTING_DRYRUN = "job.setting.dryRun";
public static final String DATAX_JOB_SETTING_PASSWD_ENCRYPT = "job.setting.passwdEncrypt";
public static final String DATAX_JOB_PREHANDLER_PLUGINTYPE = "job.preHandler.pluginType";
public static final String DATAX_JOB_PREHANDLER_PLUGINNAME = "job.preHandler.pluginName";
@ -104,6 +106,13 @@ public class CoreConstant {
public static final String DATAX_JOB_POSTHANDLER_PLUGINTYPE = "job.postHandler.pluginType";
public static final String DATAX_JOB_POSTHANDLER_PLUGINNAME = "job.postHandler.pluginName";
public static final String DATA_JOB_READER_PARAMETER_PASSWORD="job.content[0].reader.parameter.password";
public static final String DATA_JOB_WRITER_PARAMETER_PASSWORD="job.content[0].writer.parameter.password";
// ----------------------------- 局部使用的变量
public static final String JOB_WRITER = "reader";
@ -148,6 +157,8 @@ public class CoreConstant {
public static final String CURRENT_SERVICE_USERNAME = "current.service.username";
public static final String CURRENT_SERVICE_PASSWORD = "current.service.password";
public static final String PASSWD_KEY="dp.public.k";
public static final String PASSWD_KEY_DEFAULT="dwz1qaz2wsx";
// ----------------------------- 环境变量 ---------------------------------

18
core/src/test/java/ConfigurationTest.java Normal file
View File

@ -0,0 +1,18 @@
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.fastjson2.JSON;
import org.junit.Test;
import java.io.File;
/**
* @Author weizhao.dong
* @Date 2023/3/22 18:26
* @Version 1.0
*/
public class ConfigurationTest {
@Test
public void configParseTest(){
Configuration configuration=Configuration.from(new File("/Users/weizhao.dong/Documents/soft/datax_install_d/script/dwd_g2park_inout_report_s.json"));
System.out.println(JSON.toJSONString(configuration));
}
}