From 9a119ad0ff72f25022245e14e2f6572b23e0babb Mon Sep 17 00:00:00 2001 From: "mingya.wmy" Date: Wed, 23 May 2018 11:27:44 +0800 Subject: [PATCH 1/2] change the logic of getting orc column count in DFSUtil.java --- .idea/misc.xml | 13 +++++++++++++ .idea/vcs.xml | 6 ++++++ .../datax/plugin/reader/hdfsreader/DFSUtil.java | 7 +------ 3 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 .idea/misc.xml create mode 100644 .idea/vcs.xml diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..d30d09e2 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,13 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/DFSUtil.java b/hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/DFSUtil.java index 364dfead..c39d3847 100644 --- a/hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/DFSUtil.java +++ b/hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/DFSUtil.java @@ -486,15 +486,10 @@ public class DFSUtil { } private int getAllColumnsCount(String filePath) { - int columnsCount; - final String colFinal = "_col"; Path path = new Path(filePath); try { Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(hadoopConf)); - String type_struct = reader.getObjectInspector().getTypeName(); - columnsCount = (type_struct.length() - type_struct.replace(colFinal, "").length()) - / colFinal.length(); - return columnsCount; + return reader.getTypes().get(0).getSubtypesCount(); } catch (IOException e) { String message = "读取orcfile column列数失败,请联系系统管理员"; throw DataXException.asDataXException(HdfsReaderErrorCode.READ_FILE_ERROR, message); From 84cfeb51775379e0a2aba62ace62ec1f1ab2072b Mon Sep 17 00:00:00 2001 From: "mingya.wmy" Date: Wed, 23 May 2018 11:48:34 +0800 Subject: [PATCH 2/2] rm some idea files --- .idea/misc.xml | 13 ------------- .idea/vcs.xml | 6 ------ 2 files changed, 19 deletions(-) delete mode 100644 .idea/misc.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index d30d09e2..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file