github/DataX
5
0
Fork 0
mirror of https://github.com/alibaba/DataX.git synced 2025-05-03 06:09:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

support for "splitFactor" parameterization

Browse Source
This commit is contained in:
daizihao 2020-10-15 09:26:19 +08:00
parent 46a5e90fd9
commit dabb9a2fc3
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/util/ReaderSplitUtil.java
View File

@ -72,8 +72,8 @@ public final class ReaderSplitUtil {
//为避免导入hive小文件 默认基数为5，也就是channel配置几个就是几个task,可以通过 splitFactor 配置基数 //为避免导入hive小文件 默认基数为5，也就是channel配置几个就是几个task,可以通过 splitFactor 配置基数
// 最终task数为(channel/tableNum)向上取整*splitFactor // 最终task数为(channel/tableNum)向上取整*splitFactor
Integer quota = originalSliceConfig.getInt(Key.SPLIT_FACTOR, Constant.SPLIT_FACTOR); Integer splitFactor = originalSliceConfig.getInt(Key.SPLIT_FACTOR, Constant.SPLIT_FACTOR);
eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * quota; eachTableShouldSplittedNumber = eachTableShouldSplittedNumber * splitFactor;
} }
// 尝试对每个表，切分为eachTableShouldSplittedNumber 份 // 尝试对每个表，切分为eachTableShouldSplittedNumber 份
for (String table : tables) { for (String table : tables) {