github/DataX
5
0
Fork 0
mirror of https://github.com/alibaba/DataX.git synced 2025-05-03 03:40:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

support for "splitFactor" parameterization

Browse Source
This commit is contained in:
daizihao 2020-10-15 09:25:07 +08:00
parent b4a3eeb130
commit 46a5e90fd9
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/util/ReaderSplitUtil.java
View File

@ -70,8 +70,8 @@ public final class ReaderSplitUtil {
//考虑其他比率数字?(splitPk is null, 忽略此长尾) //考虑其他比率数字?(splitPk is null, 忽略此长尾)
//eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * 5; //eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * 5;
//为避免导入hive小文件 默认基数为5，也就是channel配置几个就是几个task,可以通过 pkQuota 配置基数 //为避免导入hive小文件 默认基数为5，也就是channel配置几个就是几个task,可以通过 splitFactor 配置基数
// 最终task数为(channel/tableNum)向上取整*pkQuota // 最终task数为(channel/tableNum)向上取整*splitFactor
Integer quota = originalSliceConfig.getInt(Key.SPLIT_FACTOR, Constant.SPLIT_FACTOR); Integer quota = originalSliceConfig.getInt(Key.SPLIT_FACTOR, Constant.SPLIT_FACTOR);
eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * quota; eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * quota;
} }