github/DataX
5
0
Fork 0
mirror of https://github.com/alibaba/DataX.git synced 2025-05-02 04:59:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

doris writer url decoder fix

Browse Source
This commit is contained in:
jiafeng.zhang 2023-07-31 11:47:24 +08:00
parent 82680c4c63
commit e469a8e0a7
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisStreamLoadObserver.java
View File

@ -20,6 +20,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLDecoder;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.HashMap; import java.util.HashMap;
@ -47,6 +48,18 @@ public class DorisStreamLoadObserver {
this.options = options; this.options = options;
} }
public String urlDecode(String outBuffer) {
String data = outBuffer;
try {
data = data.replaceAll("%(?![0-9a-fA-F]{2})", "%25");
data = data.replaceAll("\\+", "%2B");
data = URLDecoder.decode(data, "utf-8");
} catch (Exception e) {
e.printStackTrace();
}
return data;
}
public void streamLoad(WriterTuple data) throws Exception { public void streamLoad(WriterTuple data) throws Exception {
String host = getLoadHost(); String host = getLoadHost();
if(host == null){ if(host == null){
@ -60,6 +73,7 @@ public class DorisStreamLoadObserver {
.append("/_stream_load") .append("/_stream_load")
.toString(); .toString();
LOG.info("Start to join batch data: rows[{}] bytes[{}] label[{}].", data.getRows().size(), data.getBytes(), data.getLabel()); LOG.info("Start to join batch data: rows[{}] bytes[{}] label[{}].", data.getRows().size(), data.getBytes(), data.getLabel());
loadUrl = urlDecode(loadUrl);
Map<String, Object> loadResult = put(loadUrl, data.getLabel(), addRows(data.getRows(), data.getBytes().intValue())); Map<String, Object> loadResult = put(loadUrl, data.getLabel(), addRows(data.getRows(), data.getBytes().intValue()));
LOG.info("StreamLoad response :{}",JSON.toJSONString(loadResult)); LOG.info("StreamLoad response :{}",JSON.toJSONString(loadResult));
final String keyStatus = "Status"; final String keyStatus = "Status";