github/DataX
5
0
Fork 0
mirror of https://github.com/alibaba/DataX.git synced 2025-05-02 12:49:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

support for "splitFactor" parameterization

Browse Source
This commit is contained in:
daizihao 2020-10-14 20:39:25 +08:00
parent 809cffc5f6
commit b4a3eeb130
3 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/Constant.java
View File

@ -25,4 +25,6 @@ public final class Constant {
public static String TABLE_NAME_PLACEHOLDER = "@table";
public static Integer SPLIT_FACTOR = 5;
}

1
plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/Key.java
View File

@ -46,5 +46,6 @@ public final class Key {
public final static String DRYRUN = "dryRun";
public static String SPLIT_FACTOR = "splitFactor";
}

7
plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/util/ReaderSplitUtil.java
View File

@ -68,7 +68,12 @@ public final class ReaderSplitUtil {
//eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * 2 + 1;// 不应该加1导致长尾
//考虑其他比率数字?(splitPk is null, 忽略此长尾)
eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * 5;
//eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * 5;
//为避免导入hive小文件 默认基数为5，也就是channel配置几个就是几个task,可以通过 pkQuota 配置基数
// 最终task数为(channel/tableNum)向上取整*pkQuota
Integer quota = originalSliceConfig.getInt(Key.SPLIT_FACTOR, Constant.SPLIT_FACTOR);
eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * quota;
}
// 尝试对每个表，切分为eachTableShouldSplittedNumber 份
for (String table : tables) {