github/DataX
5
0
Fork 0
mirror of https://github.com/alibaba/DataX.git synced 2025-05-02 07:21:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge dc3a82f3d9 into 0824b45c5e

Browse Source
This commit is contained in:
lhldyf 2025-04-10 16:25:15 +08:00 committed by GitHub
commit e7824688e2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 11 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/HdfsWriter.java
View File

@ -271,21 +271,9 @@ public class HdfsWriter extends Writer {
} }
private String buildFilePath() { private String buildFilePath() {
boolean isEndWithSeparator = false; boolean isEndWithSeparator = this.path.endsWith(String.valueOf(IOUtils.DIR_SEPARATOR_UNIX));
switch (IOUtils.DIR_SEPARATOR) {
case IOUtils.DIR_SEPARATOR_UNIX:
isEndWithSeparator = this.path.endsWith(String
.valueOf(IOUtils.DIR_SEPARATOR));
break;
case IOUtils.DIR_SEPARATOR_WINDOWS:
isEndWithSeparator = this.path.endsWith(String
.valueOf(IOUtils.DIR_SEPARATOR_WINDOWS));
break;
default:
break;
}
if (!isEndWithSeparator) { if (!isEndWithSeparator) {
this.path = this.path + IOUtils.DIR_SEPARATOR; this.path = this.path + IOUtils.DIR_SEPARATOR_UNIX;
} }
return this.path; return this.path;
} }
@ -297,38 +285,18 @@ public class HdfsWriter extends Writer {
*/ */
private String buildTmpFilePath(String userPath) { private String buildTmpFilePath(String userPath) {
String tmpFilePath; String tmpFilePath;
boolean isEndWithSeparator = false; boolean isEndWithSeparator = userPath.endsWith(String.valueOf(IOUtils.DIR_SEPARATOR_UNIX));
switch (IOUtils.DIR_SEPARATOR) {
case IOUtils.DIR_SEPARATOR_UNIX:
isEndWithSeparator = userPath.endsWith(String
.valueOf(IOUtils.DIR_SEPARATOR));
break;
case IOUtils.DIR_SEPARATOR_WINDOWS:
isEndWithSeparator = userPath.endsWith(String
.valueOf(IOUtils.DIR_SEPARATOR_WINDOWS));
break;
default:
break;
}
String tmpSuffix; String tmpSuffix;
tmpSuffix = UUID.randomUUID().toString().replace('-', '_'); do {
if (!isEndWithSeparator) {
tmpFilePath = String.format("%s__%s%s", userPath, tmpSuffix, IOUtils.DIR_SEPARATOR);
}else if("/".equals(userPath)){
tmpFilePath = String.format("%s__%s%s", userPath, tmpSuffix, IOUtils.DIR_SEPARATOR);
}else{
tmpFilePath = String.format("%s__%s%s", userPath.substring(0,userPath.length()-1), tmpSuffix, IOUtils.DIR_SEPARATOR);
}
while(hdfsHelper.isPathexists(tmpFilePath)){
tmpSuffix = UUID.randomUUID().toString().replace('-', '_'); tmpSuffix = UUID.randomUUID().toString().replace('-', '_');
if (!isEndWithSeparator) { if (!isEndWithSeparator || String.valueOf(IOUtils.DIR_SEPARATOR_UNIX).equals(userPath)) {
tmpFilePath = String.format("%s__%s%s", userPath, tmpSuffix, IOUtils.DIR_SEPARATOR); tmpFilePath = String.format("%s__%s%s", userPath, tmpSuffix, IOUtils.DIR_SEPARATOR_UNIX);
}else if("/".equals(userPath)){ } else {
tmpFilePath = String.format("%s__%s%s", userPath, tmpSuffix, IOUtils.DIR_SEPARATOR); tmpFilePath = String.format("%s__%s%s", userPath.substring(0,userPath.length()-1), tmpSuffix,
}else{ IOUtils.DIR_SEPARATOR_UNIX);
tmpFilePath = String.format("%s__%s%s", userPath.substring(0,userPath.length()-1), tmpSuffix, IOUtils.DIR_SEPARATOR);
} }
} } while(hdfsHelper.isPathexists(tmpFilePath));
return tmpFilePath; return tmpFilePath;
} }
public void unitizeParquetConfig(Configuration writerSliceConfig) { public void unitizeParquetConfig(Configuration writerSliceConfig) {