github/DataX
5
0
Fork 0
mirror of https://github.com/alibaba/DataX.git synced 2025-05-02 23:52:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

change the logic of getting orc column count in DFSUtil.java

Browse Source
This commit is contained in:
mingya.wmy 2018-05-23 11:27:44 +08:00 committed by windpiger
parent 4608abe947
commit 9a119ad0ff
3 changed files with 20 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.idea/misc.xml Normal file
View File

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/classes" />
</component>
</project>

6
.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

7
hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/DFSUtil.java
View File

@ -486,15 +486,10 @@ public class DFSUtil {
} }
private int getAllColumnsCount(String filePath) { private int getAllColumnsCount(String filePath) {
int columnsCount;
final String colFinal = "_col";
Path path = new Path(filePath); Path path = new Path(filePath);
try { try {
Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(hadoopConf)); Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(hadoopConf));
String type_struct = reader.getObjectInspector().getTypeName(); return reader.getTypes().get(0).getSubtypesCount();
columnsCount = (type_struct.length() - type_struct.replace(colFinal, "").length())
/ colFinal.length();
return columnsCount;
} catch (IOException e) { } catch (IOException e) {
String message = "读取orcfile column列数失败，请联系系统管理员"; String message = "读取orcfile column列数失败，请联系系统管理员";
throw DataXException.asDataXException(HdfsReaderErrorCode.READ_FILE_ERROR, message); throw DataXException.asDataXException(HdfsReaderErrorCode.READ_FILE_ERROR, message);