github/DataX
5
0
Fork 0
mirror of https://github.com/alibaba/DataX.git synced 2025-05-03 03:40:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #262 from wangchuande/hotfix/hdfs-reader-split

Browse Source 
fix orcFileStartRead
This commit is contained in:
dingxiaobo 2023-03-23 13:46:57 +08:00 committed by GitHub
commit 430f08b16a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 20 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/DFSUtil.java
View File

@ -331,9 +331,11 @@ public class DFSUtil {
//If the network disconnected, will retry 45 times, each time the retry interval for 20 seconds //If the network disconnected, will retry 45 times, each time the retry interval for 20 seconds
//Each file as a split //Each file as a split
//TODO multy threads //TODO multy threads
InputSplit[] splits = in.getSplits(conf, 1); // OrcInputFormat getSplits params numSplits not used, splits size = block numbers
InputSplit[] splits = in.getSplits(conf, -1);
RecordReader reader = in.getRecordReader(splits[0], conf, Reporter.NULL); for (InputSplit split : splits) {
{
RecordReader reader = in.getRecordReader(split, conf, Reporter.NULL);
Object key = reader.createKey(); Object key = reader.createKey();
Object value = reader.createValue(); Object value = reader.createValue();
// 获取列信息 // 获取列信息
@ -351,6 +353,8 @@ public class DFSUtil {
taskPluginCollector, isReadAllColumns, nullFormat); taskPluginCollector, isReadAllColumns, nullFormat);
} }
reader.close(); reader.close();
}
}
} catch (Exception e) { } catch (Exception e) {
String message = String.format("从orcfile文件路径[%s]中读取数据发生异常，请联系系统管理员。" String message = String.format("从orcfile文件路径[%s]中读取数据发生异常，请联系系统管理员。"
, sourceOrcFilePath); , sourceOrcFilePath);