configurationList = config.getListConfiguration(Key.COLUMN);
+ for (Configuration column : configurationList) {
+ ColumnType columnType;
+ try {
+ columnType = ColumnType.valueOf(column.getString(Key.COLUMN_NODE_TYPE));
+ } catch (NullPointerException | IllegalArgumentException e) {
+ throw DataXException.asDataXException(GdbReaderErrorCode.BAD_CONFIG_VALUE, Key.COLUMN_NODE_TYPE);
+ }
+
+ if (exportType == ExportType.VERTEX) {
+ // only id/label/property column allow when vertex
+ ConfigHelper.assertConfig(Key.COLUMN_NODE_TYPE, () ->
+ columnType == ColumnType.primaryKey || columnType == ColumnType.primaryLabel
+ || columnType == ColumnType.vertexProperty || columnType == ColumnType.vertexJsonProperty);
+ } else if (exportType == ExportType.EDGE) {
+ // edge
+ ConfigHelper.assertConfig(Key.COLUMN_NODE_TYPE, () ->
+ columnType == ColumnType.primaryKey || columnType == ColumnType.primaryLabel
+ || columnType == ColumnType.srcPrimaryKey || columnType == ColumnType.srcPrimaryLabel
+ || columnType == ColumnType.dstPrimaryKey || columnType == ColumnType.dstPrimaryLabel
+ || columnType == ColumnType.edgeProperty || columnType == ColumnType.edgeJsonProperty);
+ }
+
+ if (columnType == ColumnType.edgeProperty || columnType == ColumnType.vertexProperty) {
+ String name = column.getString(Key.COLUMN_NAME);
+ ValueType propType = ValueType.fromShortName(column.getString(Key.COLUMN_TYPE));
+
+ ConfigHelper.assertConfig(Key.COLUMN_NAME, () -> name != null);
+ if (propType == null) {
+ throw DataXException.asDataXException(GdbReaderErrorCode.UNSUPPORTED_TYPE, Key.COLUMN_TYPE);
+ }
+ rule.addColumn(columnType, propType, name);
+ } else if (columnType == ColumnType.vertexJsonProperty || columnType == ColumnType.edgeJsonProperty) {
+ rule.addJsonColumn(columnType);
+ } else {
+ rule.addColumn(columnType, ValueType.STRING, null);
+ }
+ }
+ return rule;
+ }
+}
diff --git a/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/mapping/ValueType.java b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/mapping/ValueType.java
new file mode 100644
index 00000000..826e0493
--- /dev/null
+++ b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/mapping/ValueType.java
@@ -0,0 +1,128 @@
+/*
+ * (C) 2019-present Alibaba Group Holding Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+package com.alibaba.datax.plugin.reader.gdbreader.mapping;
+
+import com.alibaba.datax.common.element.BoolColumn;
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.element.DoubleColumn;
+import com.alibaba.datax.common.element.LongColumn;
+import com.alibaba.datax.common.element.StringColumn;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * @author : Liu Jianping
+ * @date : 2019/9/6
+ */
+
+public enum ValueType {
+ /**
+ * transfer gdb element object value to DataX Column data
+ *
+ * int, long -> LongColumn
+ * float, double -> DoubleColumn
+ * bool -> BooleanColumn
+ * string -> StringColumn
+ */
+ INT(Integer.class, "int", ValueTypeHolder::longColumnMapper),
+ INTEGER(Integer.class, "integer", ValueTypeHolder::longColumnMapper),
+ LONG(Long.class, "long", ValueTypeHolder::longColumnMapper),
+ DOUBLE(Double.class, "double", ValueTypeHolder::doubleColumnMapper),
+ FLOAT(Float.class, "float", ValueTypeHolder::doubleColumnMapper),
+ BOOLEAN(Boolean.class, "boolean", ValueTypeHolder::boolColumnMapper),
+ STRING(String.class, "string", ValueTypeHolder::stringColumnMapper),
+ ;
+
+ private Class type = null;
+ private String shortName = null;
+ private Function columnFunc = null;
+
+ ValueType(Class type, String name, Function columnFunc) {
+ this.type = type;
+ this.shortName = name;
+ this.columnFunc = columnFunc;
+
+ ValueTypeHolder.shortName2type.put(shortName, this);
+ }
+
+ public static ValueType fromShortName(String name) {
+ return ValueTypeHolder.shortName2type.get(name);
+ }
+
+ public Column applyObject(Object value) {
+ if (value == null) {
+ return null;
+ }
+ return columnFunc.apply(value);
+ }
+
+ private static class ValueTypeHolder {
+ private static Map shortName2type = new HashMap<>();
+
+ private static LongColumn longColumnMapper(Object o) {
+ long v;
+ if (o instanceof Integer) {
+ v = (int) o;
+ } else if (o instanceof Long) {
+ v = (long) o;
+ } else if (o instanceof String) {
+ v = Long.valueOf((String) o);
+ } else {
+ throw new RuntimeException("Failed to cast " + o.getClass() + " to Long");
+ }
+
+ return new LongColumn(v);
+ }
+
+ private static DoubleColumn doubleColumnMapper(Object o) {
+ double v;
+ if (o instanceof Integer) {
+ v = (double) (int) o;
+ } else if (o instanceof Long) {
+ v = (double) (long) o;
+ } else if (o instanceof Float) {
+ v = (double) (float) o;
+ } else if (o instanceof Double) {
+ v = (double) o;
+ } else if (o instanceof String) {
+ v = Double.valueOf((String) o);
+ } else {
+ throw new RuntimeException("Failed to cast " + o.getClass() + " to Double");
+ }
+
+ return new DoubleColumn(v);
+ }
+
+ private static BoolColumn boolColumnMapper(Object o) {
+ boolean v;
+ if (o instanceof Integer) {
+ v = ((int) o != 0);
+ } else if (o instanceof Long) {
+ v = ((long) o != 0);
+ } else if (o instanceof Boolean) {
+ v = (boolean) o;
+ } else if (o instanceof String) {
+ v = Boolean.valueOf((String) o);
+ } else {
+ throw new RuntimeException("Failed to cast " + o.getClass() + " to Boolean");
+ }
+
+ return new BoolColumn(v);
+ }
+
+ private static StringColumn stringColumnMapper(Object o) {
+ if (o instanceof String) {
+ return new StringColumn((String) o);
+ } else {
+ return new StringColumn(String.valueOf(o));
+ }
+ }
+ }
+}
diff --git a/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/AbstractGdbGraph.java b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/AbstractGdbGraph.java
new file mode 100644
index 00000000..4eda2eed
--- /dev/null
+++ b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/AbstractGdbGraph.java
@@ -0,0 +1,89 @@
+/*
+ * (C) 2019-present Alibaba Group Holding Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+package com.alibaba.datax.plugin.reader.gdbreader.model;
+
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.reader.gdbreader.Key;
+import org.apache.tinkerpop.gremlin.driver.Client;
+import org.apache.tinkerpop.gremlin.driver.Cluster;
+import org.apache.tinkerpop.gremlin.driver.RequestOptions;
+import org.apache.tinkerpop.gremlin.driver.Result;
+import org.apache.tinkerpop.gremlin.driver.ResultSet;
+import org.apache.tinkerpop.gremlin.driver.ser.Serializers;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author : Liu Jianping
+ * @date : 2019/9/6
+ */
+
+public abstract class AbstractGdbGraph implements GdbGraph {
+ final static int DEFAULT_TIMEOUT = 30000;
+ private static final Logger log = LoggerFactory.getLogger(AbstractGdbGraph.class);
+ private Client client;
+
+ AbstractGdbGraph() {
+ }
+
+ AbstractGdbGraph(Configuration config) {
+ log.info("init graphdb client");
+ String host = config.getString(Key.HOST);
+ int port = config.getInt(Key.PORT);
+ String username = config.getString(Key.USERNAME);
+ String password = config.getString(Key.PASSWORD);
+
+ try {
+ Cluster cluster = Cluster.build(host).port(port).credentials(username, password)
+ .serializer(Serializers.GRAPHBINARY_V1D0)
+ .maxContentLength(1024 * 1024)
+ .resultIterationBatchSize(64)
+ .create();
+ client = cluster.connect().init();
+
+ warmClient();
+ } catch (RuntimeException e) {
+ log.error("Failed to connect to GDB {}:{}, due to {}", host, port, e);
+ throw e;
+ }
+ }
+
+ protected List runInternal(String dsl, Map params) throws Exception {
+ return runInternalAsync(dsl, params).all().get(DEFAULT_TIMEOUT + 1000, TimeUnit.MILLISECONDS);
+ }
+
+ protected ResultSet runInternalAsync(String dsl, Map params) throws Exception {
+ RequestOptions.Builder options = RequestOptions.build().timeout(DEFAULT_TIMEOUT);
+ if (params != null && !params.isEmpty()) {
+ params.forEach(options::addParameter);
+ }
+ return client.submitAsync(dsl, options.create()).get(DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS);
+ }
+
+ private void warmClient() {
+ try {
+ runInternal("g.V('test')", null);
+ log.info("warm graphdb client over");
+ } catch (Exception e) {
+ log.error("warmClient error");
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (client != null) {
+ log.info("close graphdb client");
+ client.close();
+ }
+ }
+}
diff --git a/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/GdbElement.java b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/GdbElement.java
new file mode 100644
index 00000000..79619ad0
--- /dev/null
+++ b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/GdbElement.java
@@ -0,0 +1,39 @@
+/*
+ * (C) 2019-present Alibaba Group Holding Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+package com.alibaba.datax.plugin.reader.gdbreader.model;
+
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author : Liu Jianping
+ * @date : 2019/9/6
+ */
+
+@Data
+public class GdbElement {
+ String id = null;
+ String label = null;
+ String to = null;
+ String from = null;
+ String toLabel = null;
+ String fromLabel = null;
+
+ Map properties = new HashMap<>();
+
+ public GdbElement() {
+ }
+
+ public GdbElement(String id, String label) {
+ this.id = id;
+ this.label = label;
+ }
+
+}
diff --git a/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/GdbGraph.java b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/GdbGraph.java
new file mode 100644
index 00000000..e6651293
--- /dev/null
+++ b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/GdbGraph.java
@@ -0,0 +1,65 @@
+/*
+ * (C) 2019-present Alibaba Group Holding Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+package com.alibaba.datax.plugin.reader.gdbreader.model;
+
+import org.apache.tinkerpop.gremlin.driver.ResultSet;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author : Liu Jianping
+ * @date : 2019/9/6
+ */
+
+public interface GdbGraph extends AutoCloseable {
+
+ /**
+ * Get All labels of GraphDB
+ *
+ * @return labels map included numbers
+ */
+ Map getLabels();
+
+ /**
+ * Get the Ids list of special 'label', size up to 'limit'
+ *
+ * @param label is Label of Vertex or Edge
+ * @param start of Ids range to get
+ * @param limit size of Ids list
+ * @return Ids list
+ */
+ List fetchIds(String label, String start, long limit);
+
+ /**
+ * Fetch element in async mode, just send query dsl to server
+ *
+ * @param label node label to filter
+ * @param start range begin(included)
+ * @param end range end(included)
+ * @param propNames propKey list to fetch
+ * @return future to get result later
+ */
+ ResultSet fetchElementsAsync(String label, String start, String end, List propNames);
+
+ /**
+ * Get get element from Response @{ResultSet}
+ *
+ * @param results Response of Server
+ * @return element sets
+ */
+ List getElement(ResultSet results);
+
+ /**
+ * close graph client
+ *
+ * @throws Exception if fails
+ */
+ @Override
+ void close() throws Exception;
+}
diff --git a/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/ScriptGdbGraph.java b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/ScriptGdbGraph.java
new file mode 100644
index 00000000..8c08b819
--- /dev/null
+++ b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/ScriptGdbGraph.java
@@ -0,0 +1,192 @@
+/*
+ * (C) 2019-present Alibaba Group Holding Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+package com.alibaba.datax.plugin.reader.gdbreader.model;
+
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.reader.gdbreader.Key.ExportType;
+import org.apache.tinkerpop.gremlin.driver.Result;
+import org.apache.tinkerpop.gremlin.driver.ResultSet;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceEdge;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author : Liu Jianping
+ * @date : 2019/9/6
+ */
+
+public class ScriptGdbGraph extends AbstractGdbGraph {
+ private static final Logger log = LoggerFactory.getLogger(ScriptGdbGraph.class);
+
+ private final static String LABEL = "GDB___LABEL";
+ private final static String START_ID = "GDB___ID";
+ private final static String END_ID = "GDB___ID_END";
+ private final static String LIMIT = "GDB___LIMIT";
+
+ private final static String FETCH_VERTEX_IDS_DSL = "g.V().hasLabel(" + LABEL + ").has(id, gt(" + START_ID + ")).limit(" + LIMIT + ").id()";
+ private final static String FETCH_EDGE_IDS_DSL = "g.E().hasLabel(" + LABEL + ").has(id, gt(" + START_ID + ")).limit(" + LIMIT + ").id()";
+
+ private final static String FETCH_VERTEX_LABELS_DSL = "g.V().groupCount().by(label)";
+ private final static String FETCH_EDGE_LABELS_DSL = "g.E().groupCount().by(label)";
+
+ /**
+ * fetch node range [START_ID, END_ID]
+ */
+ private final static String FETCH_RANGE_VERTEX_DSL = "g.V().hasLabel(" + LABEL + ").has(id, gte(" + START_ID + ")).has(id, lte(" + END_ID + "))";
+ private final static String FETCH_RANGE_EDGE_DSL = "g.E().hasLabel(" + LABEL + ").has(id, gte(" + START_ID + ")).has(id, lte(" + END_ID + "))";
+ private final static String PART_WITH_PROP_DSL = ".as('a').project('node', 'props').by(select('a')).by(select('a').propertyMap(";
+
+ private final ExportType exportType;
+
+ public ScriptGdbGraph(ExportType exportType) {
+ super();
+ this.exportType = exportType;
+ }
+
+ public ScriptGdbGraph(Configuration config, ExportType exportType) {
+ super(config);
+ this.exportType = exportType;
+ }
+
+ @Override
+ public List fetchIds(final String label, final String start, long limit) {
+ Map params = new HashMap(3) {{
+ put(LABEL, label);
+ put(START_ID, start);
+ put(LIMIT, limit);
+ }};
+ String fetchDsl = exportType == ExportType.VERTEX ? FETCH_VERTEX_IDS_DSL : FETCH_EDGE_IDS_DSL;
+
+ List ids = new ArrayList<>();
+ try {
+ List results = runInternal(fetchDsl, params);
+
+ // transfer result to id string
+ results.forEach(id -> ids.add(id.getString()));
+ } catch (Exception e) {
+ log.error("fetch range node failed, label {}, start {}", label, start);
+ throw new RuntimeException(e);
+ }
+ return ids;
+ }
+
+ @Override
+ public ResultSet fetchElementsAsync(final String label, final String start, final String end, final List propNames) {
+ Map params = new HashMap<>(3);
+ params.put(LABEL, label);
+ params.put(START_ID, start);
+ params.put(END_ID, end);
+
+ String prefixDsl = exportType == ExportType.VERTEX ? FETCH_RANGE_VERTEX_DSL : FETCH_RANGE_EDGE_DSL;
+ StringBuilder fetchDsl = new StringBuilder(prefixDsl);
+ if (propNames != null) {
+ fetchDsl.append(PART_WITH_PROP_DSL);
+ for (int i = 0; i < propNames.size(); i++) {
+ String propName = "GDB___PK" + String.valueOf(i);
+ params.put(propName, propNames.get(i));
+
+ fetchDsl.append(propName);
+ if (i != propNames.size() - 1) {
+ fetchDsl.append(", ");
+ }
+ }
+ fetchDsl.append("))");
+ }
+
+ try {
+ return runInternalAsync(fetchDsl.toString(), params);
+ } catch (Exception e) {
+ log.error("Failed to fetch range node startId {}, end {} , e {}", start, end, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public List getElement(ResultSet results) {
+ List elements = new LinkedList<>();
+ try {
+ List resultList = results.all().get(DEFAULT_TIMEOUT + 1000, TimeUnit.MILLISECONDS);
+
+ resultList.forEach(n -> {
+ Object o = n.getObject();
+ GdbElement element = new GdbElement();
+ if (o instanceof Map) {
+ // project response
+ Object node = ((Map) o).get("node");
+ Object props = ((Map) o).get("props");
+
+ mapNodeToElement(node, element);
+ mapPropToElement((Map) props, element);
+ } else {
+ // range node response
+ mapNodeToElement(n.getObject(), element);
+ }
+ if (element.getId() != null) {
+ elements.add(element);
+ }
+ });
+ } catch (Exception e) {
+ log.error("Failed to get node: {}", e);
+ throw new RuntimeException(e);
+ }
+ return elements;
+ }
+
+ private void mapNodeToElement(Object node, GdbElement element) {
+ if (node instanceof ReferenceVertex) {
+ ReferenceVertex v = (ReferenceVertex) node;
+
+ element.setId((String) v.id());
+ element.setLabel(v.label());
+ } else if (node instanceof ReferenceEdge) {
+ ReferenceEdge e = (ReferenceEdge) node;
+
+ element.setId((String) e.id());
+ element.setLabel(e.label());
+ element.setTo((String) e.inVertex().id());
+ element.setToLabel(e.inVertex().label());
+ element.setFrom((String) e.outVertex().id());
+ element.setFromLabel(e.outVertex().label());
+ }
+ }
+
+ private void mapPropToElement(Map props, GdbElement element) {
+ element.setProperties(props);
+ }
+
+ @Override
+ public Map getLabels() {
+ String dsl = exportType == ExportType.VERTEX ? FETCH_VERTEX_LABELS_DSL : FETCH_EDGE_LABELS_DSL;
+
+ try {
+ List results = runInternal(dsl, null);
+ Map labelMap = new HashMap<>(2);
+
+ Map labels = results.get(0).get(Map.class);
+ labels.forEach((k, v) -> {
+ String label = (String) k;
+ Long count = (Long) v;
+ labelMap.put(label, count);
+ });
+
+ return labelMap;
+ } catch (Exception e) {
+ log.error("Failed to fetch label list, please give special labels and run again, e {}", e);
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/util/ConfigHelper.java b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/util/ConfigHelper.java
new file mode 100644
index 00000000..2ec9d153
--- /dev/null
+++ b/gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/util/ConfigHelper.java
@@ -0,0 +1,77 @@
+/*
+ * (C) 2019-present Alibaba Group Holding Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+package com.alibaba.datax.plugin.reader.gdbreader.util;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.reader.gdbreader.GdbReaderErrorCode;
+import com.alibaba.datax.plugin.reader.gdbreader.Key;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+/**
+ * @author : Liu Jianping
+ * @date : 2019/9/6
+ */
+
+public interface ConfigHelper {
+ static void assertConfig(String key, Supplier f) {
+ if (!f.get()) {
+ throw DataXException.asDataXException(GdbReaderErrorCode.BAD_CONFIG_VALUE, key);
+ }
+ }
+
+ static void assertHasContent(Configuration config, String key) {
+ assertConfig(key, () -> StringUtils.isNotBlank(config.getString(key)));
+ }
+
+ static void assertGdbClient(Configuration config) {
+ assertHasContent(config, Key.HOST);
+ assertConfig(Key.PORT, () -> config.getInt(Key.PORT) > 0);
+
+ assertHasContent(config, Key.USERNAME);
+ assertHasContent(config, Key.PASSWORD);
+ }
+
+ static List assertLabels(Configuration config) {
+ Object labels = config.get(Key.LABEL);
+ if (!(labels instanceof List)) {
+ throw DataXException.asDataXException(GdbReaderErrorCode.BAD_CONFIG_VALUE, "labels should be List");
+ }
+
+ List list = (List) labels;
+ List configLabels = new ArrayList<>(0);
+ list.forEach(n -> configLabels.add(String.valueOf(n)));
+
+ return configLabels;
+ }
+
+ static List splitConfig(Configuration config, List labels) {
+ List configs = new ArrayList<>();
+ for (String label : labels) {
+ Configuration conf = config.clone();
+ conf.set(Key.LABEL, label);
+
+ configs.add(conf);
+ }
+ return configs;
+ }
+
+ static Configuration fromClasspath(String name) {
+ try (InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(name)) {
+ return Configuration.from(is);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("File not found: " + name);
+ }
+ }
+}
diff --git a/gdbreader/src/main/resources/plugin.json b/gdbreader/src/main/resources/plugin.json
new file mode 100644
index 00000000..4fedfa24
--- /dev/null
+++ b/gdbreader/src/main/resources/plugin.json
@@ -0,0 +1,6 @@
+{
+ "name": "gdbreader",
+ "class": "com.alibaba.datax.plugin.reader.gdbreader.GdbReader",
+ "description": "useScene: prod. mechanism: connect GDB with gremlin-client, execute 'g.V().propertyMap() or g.E().propertyMap()' to get record",
+ "developer": "alibaba"
+}
\ No newline at end of file
diff --git a/gdbreader/src/main/resources/plugin_job_template.json b/gdbreader/src/main/resources/plugin_job_template.json
new file mode 100644
index 00000000..bfca7780
--- /dev/null
+++ b/gdbreader/src/main/resources/plugin_job_template.json
@@ -0,0 +1,77 @@
+{
+ "job": {
+ "setting": {
+ "speed": {
+ "channel": 1
+ },
+ "errorLimit": {
+ "record": 1
+ }
+ },
+
+ "content": [
+ {
+ "reader": {
+ "name": "gdbreader",
+ "parameter": {
+ "host": "10.218.145.24",
+ "port": 8182,
+ "username": "***",
+ "password": "***",
+ "labelType": "EDGE",
+ "labels": ["label1", "label2"],
+ "column": [
+ {
+ "name": "id",
+ "type": "string",
+ "columnType": "primaryKey"
+ },
+ {
+ "name": "label",
+ "type": "string",
+ "columnType": "primaryLabel"
+ },
+ {
+ "name": "srcId",
+ "type": "string",
+ "columnType": "srcPrimaryKey"
+ },
+ {
+ "name": "srcLabel",
+ "type": "string",
+ "columnType": "srcPrimaryLabel"
+ },
+ {
+ "name": "dstId",
+ "type": "string",
+ "columnType": "srcPrimaryKey"
+ },
+ {
+ "name": "dstLabel",
+ "type": "string",
+ "columnType": "srcPrimaryLabel"
+ },
+ {
+ "name": "name",
+ "type": "string",
+ "columnType": "edgeProperty"
+ },
+ {
+ "name": "weight",
+ "type": "double",
+ "columnType": "edgeProperty"
+ }
+ ]
+ }
+ },
+
+ "writer": {
+ "name": "streamwriter",
+ "parameter": {
+ "print": true
+ }
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/gdbwriter/doc/gdbwriter.md b/gdbwriter/doc/gdbwriter.md
index 82cdd899..8c1e11e6 100644
--- a/gdbwriter/doc/gdbwriter.md
+++ b/gdbwriter/doc/gdbwriter.md
@@ -41,6 +41,14 @@ GDBWriter通过DataX框架获取Reader生成的协议数据，使用`g.addV/E(GD
{
"random": "60,64",
"type": "string"
+ },
+ {
+ "random": "100,1000",
+ "type": "long"
+ },
+ {
+ "random": "32,48",
+ "type": "string"
}
],
"sliceRecordCount": 1000
@@ -55,20 +63,32 @@ GDBWriter通过DataX框架获取Reader生成的协议数据，使用`g.addV/E(GD
"password": "***",
"writeMode": "INSERT",
"labelType": "VERTEX",
- "label": "${1}",
+ "label": "#{1}",
"idTransRule": "none",
"session": true,
"maxRecordsInBatch": 64,
"column": [
{
"name": "id",
- "value": "${0}",
+ "value": "#{0}",
"type": "string",
"columnType": "primaryKey"
},
{
"name": "vertex_propKey",
- "value": "${2}",
+ "value": "#{2}",
+ "type": "string",
+ "columnType": "vertexSetProperty"
+ },
+ {
+ "name": "vertex_propKey",
+ "value": "#{3}",
+ "type": "long",
+ "columnType": "vertexSetProperty"
+ },
+ {
+ "name": "vertex_propKey2",
+ "value": "#{4}",
"type": "string",
"columnType": "vertexProperty"
}
@@ -134,7 +154,7 @@ GDBWriter通过DataX框架获取Reader生成的协议数据，使用`g.addV/E(GD
"password": "***",
"writeMode": "INSERT",
"labelType": "EDGE",
- "label": "${3}",
+ "label": "#{3}",
"idTransRule": "none",
"srcIdTransRule": "labelPrefix",
"dstIdTransRule": "labelPrefix",
@@ -144,25 +164,25 @@ GDBWriter通过DataX框架获取Reader生成的协议数据，使用`g.addV/E(GD
"column": [
{
"name": "id",
- "value": "${0}",
+ "value": "#{0}",
"type": "string",
"columnType": "primaryKey"
},
{
"name": "id",
- "value": "${1}",
+ "value": "#{1}",
"type": "string",
"columnType": "srcPrimaryKey"
},
{
"name": "id",
- "value": "${2}",
+ "value": "#{2}",
"type": "string",
"columnType": "dstPrimaryKey"
},
{
"name": "edge_propKey",
- "value": "${4}",
+ "value": "#{4}",
"type": "string",
"columnType": "edgeProperty"
}
@@ -199,7 +219,7 @@ GDBWriter通过DataX框架获取Reader生成的协议数据，使用`g.addV/E(GD
* 默认值：无
* **label**
- * 描述：类型名，即点/边名称； label支持从源列中读取，如${0}，表示取第一列字段作为label名。源列索引从0开始；
+ * 描述：类型名，即点/边名称； label支持从源列中读取，如#{0}，表示取第一列字段作为label名。源列索引从0开始；
* 必选：是
* 默认值：无
@@ -211,12 +231,12 @@ GDBWriter通过DataX框架获取Reader生成的协议数据，使用`g.addV/E(GD
* 默认值：无
* **srcLabel**
- * 描述：当label为边时，表示起点的点名称；srcLabel支持从源列中读取，如${0}，表示取第一列字段作为label名。源列索引从0开始；
+ * 描述：当label为边时，表示起点的点名称；srcLabel支持从源列中读取，如#{0}，表示取第一列字段作为label名。源列索引从0开始；
* 必选：labelType为边，srcIdTransRule为none时可不填写，否则必填；
* 默认值：无
* **dstLabel**
- * 描述：当label为边时，表示终点的点名称；dstLabel支持从源列中读取，如${0}，表示取第一列字段作为label名。源列索引从0开始；
+ * 描述：当label为边时，表示终点的点名称；dstLabel支持从源列中读取，如#{0}，表示取第一列字段作为label名。源列索引从0开始；
* 必选：labelType为边，dstIdTransRule为none时可不填写，否则必填；
* 默认值：无
@@ -271,9 +291,9 @@ GDBWriter通过DataX框架获取Reader生成的协议数据，使用`g.addV/E(GD
* **column -> value**
* 描述：点/边映射关系的字段值；
- * ${N}表示直接映射源端值，N为源端column索引，从0开始；${0}表示映射源端column第1个字段；
- * test-${0} 表示源端值做拼接转换，${0}值前/后可添加固定字符串;
- * ${0}-${1}表示做多字段拼接，也可在任意位置添加固定字符串，如test-${0}-test1-${1}-test2
+ * #{N}表示直接映射源端值，N为源端column索引，从0开始；#{0}表示映射源端column第1个字段；
+ * test-#{0} 表示源端值做拼接转换，#{0}值前/后可添加固定字符串;
+ * #{0}-#{1}表示做多字段拼接，也可在任意位置添加固定字符串，如test-#{0}-test1-#{1}-test2
* 必选：是
* 默认值：无
@@ -290,6 +310,7 @@ GDBWriter通过DataX框架获取Reader生成的协议数据，使用`g.addV/E(GD
* primaryKey：表示该字段是主键id
* 点枚举值：
* vertexProperty：labelType为点时，表示该字段是点的普通属性
+ * vertexSetProperty：labelType为点时，表示该字段是点的SET属性，value是SET属性中的一个属性值
* vertexJsonProperty：labelType为点时，表示是点json属性，value结构请见备注**json properties示例**，点配置最多只允许出现一个json属性；
* 边枚举值：
* srcPrimaryKey：labelType为边时，表示该字段是起点主键id
@@ -305,6 +326,14 @@ GDBWriter通过DataX框架获取Reader生成的协议数据，使用`g.addV/E(GD
> {"k":"age","t":"int","v":"20"},
> {"k":"sex","t":"string","v":"male"}
> ]}
+ >
+ > # json格式同样支持给点添加SET属性，格式如下
+ > {"properties":[
+ > {"k":"name","t":"string","v":"tom","c":"set"},
+ > {"k":"name","t":"string","v":"jack","c":"set"},
+ > {"k":"age","t":"int","v":"20"},
+ > {"k":"sex","t":"string","v":"male"}
+ > ]}
> ```
## 4 性能报告
@@ -367,4 +396,5 @@ DataX压测机器
- GDBWriter插件与用户查询DSL使用相同的GDB实例端口，导入时可能会影响查询性能
## FAQ
-无
+1. 使用SET属性需要升级GDB实例到`1.0.20`版本及以上。
+2. 边只支持普通单值属性，不能给边写SET属性数据。
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriter.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriter.java
index 753d89fc..6470e9e6 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriter.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriter.java
@@ -1,10 +1,5 @@
package com.alibaba.datax.plugin.writer.gdbwriter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.*;
-import java.util.function.Function;
-
import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordReceiver;
@@ -18,24 +13,33 @@ import com.alibaba.datax.plugin.writer.gdbwriter.mapping.MappingRule;
import com.alibaba.datax.plugin.writer.gdbwriter.mapping.MappingRuleFactory;
import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbElement;
import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbGraph;
-
import groovy.lang.Tuple2;
import io.netty.util.concurrent.DefaultThreadFactory;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class GdbWriter extends Writer {
- private static final Logger log = LoggerFactory.getLogger(GdbWriter.class);
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
- private static Function mapper = null;
- private static GdbGraph globalGraph = null;
- private static boolean session = false;
+public class GdbWriter extends Writer {
+ private static final Logger log = LoggerFactory.getLogger(GdbWriter.class);
+
+ private static Function mapper = null;
+ private static GdbGraph globalGraph = null;
+ private static boolean session = false;
/**
* Job 中的方法仅执行一次，Task 中方法会由框架启动多个 Task 线程并行执行。
*
* 整个 Writer 执行流程是：
+ *
*
* Job类init-->prepare-->split
*
@@ -46,17 +50,16 @@ public class GdbWriter extends Writer {
*
*/
public static class Job extends Writer.Job {
- private static final Logger LOG = LoggerFactory
- .getLogger(Job.class);
+ private static final Logger LOG = LoggerFactory.getLogger(Job.class);
private Configuration jobConfig = null;
-
+
@Override
public void init() {
- LOG.info("GDB datax plugin writer job init begin ...");
- this.jobConfig = getPluginJobConf();
- GdbWriterConfig.of(this.jobConfig);
- LOG.info("GDB datax plugin writer job init end.");
+ LOG.info("GDB datax plugin writer job init begin ...");
+ this.jobConfig = getPluginJobConf();
+ GdbWriterConfig.of(this.jobConfig);
+ LOG.info("GDB datax plugin writer job init end.");
/**
* 注意：此方法仅执行一次。
@@ -71,37 +74,37 @@ public class GdbWriter extends Writer {
* 注意：此方法仅执行一次。
* 最佳实践：如果 Job 中有需要进行数据同步之前的处理，可以在此处完成，如果没有必要则可以直接去掉。
*/
- super.prepare();
+ super.prepare();
- MappingRule rule = MappingRuleFactory.getInstance().createV2(jobConfig);
+ final MappingRule rule = MappingRuleFactory.getInstance().createV2(this.jobConfig);
- mapper = new DefaultGdbMapper().getMapper(rule);
- session = jobConfig.getBool(Key.SESSION_STATE, false);
+ mapper = new DefaultGdbMapper(this.jobConfig).getMapper(rule);
+ session = this.jobConfig.getBool(Key.SESSION_STATE, false);
/**
* client connect check before task
*/
try {
- globalGraph = GdbGraphManager.instance().getGraph(jobConfig, false);
- } catch (RuntimeException e) {
+ globalGraph = GdbGraphManager.instance().getGraph(this.jobConfig, false);
+ } catch (final RuntimeException e) {
throw DataXException.asDataXException(GdbWriterErrorCode.FAIL_CLIENT_CONNECT, e.getMessage());
}
}
@Override
- public List split(int mandatoryNumber) {
+ public List split(final int mandatoryNumber) {
/**
* 注意：此方法仅执行一次。
* 最佳实践：通常采用工具静态类完成把 Job 配置切分成多个 Task 配置的工作。
* 这里的 mandatoryNumber 是强制必须切分的份数。
*/
- LOG.info("split begin...");
- List configurationList = new ArrayList();
- for (int i = 0; i < mandatoryNumber; i++) {
- configurationList.add(this.jobConfig.clone());
- }
- LOG.info("split end...");
- return configurationList;
+ LOG.info("split begin...");
+ final List configurationList = new ArrayList();
+ for (int i = 0; i < mandatoryNumber; i++) {
+ configurationList.add(this.jobConfig.clone());
+ }
+ LOG.info("split end...");
+ return configurationList;
}
@Override
@@ -127,7 +130,7 @@ public class GdbWriter extends Writer {
public static class Task extends Writer.Task {
private Configuration taskConfig;
-
+
private int failed = 0;
private int batchRecords;
private ExecutorService submitService = null;
@@ -139,24 +142,24 @@ public class GdbWriter extends Writer {
* 注意：此方法每个 Task 都会执行一次。
* 最佳实践：此处通过对 taskConfig 配置的读取，进而初始化一些资源为 startWrite()做准备。
*/
- this.taskConfig = super.getPluginJobConf();
- batchRecords = taskConfig.getInt(Key.MAX_RECORDS_IN_BATCH, GdbWriterConfig.DEFAULT_RECORD_NUM_IN_BATCH);
- submitService = new ThreadPoolExecutor(1, 1, 0L,
- TimeUnit.MILLISECONDS, new LinkedBlockingDeque<>(), new DefaultThreadFactory("submit-dsl"));
+ this.taskConfig = super.getPluginJobConf();
+ this.batchRecords = this.taskConfig.getInt(Key.MAX_RECORDS_IN_BATCH, GdbWriterConfig.DEFAULT_RECORD_NUM_IN_BATCH);
+ this.submitService = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingDeque<>(),
+ new DefaultThreadFactory("submit-dsl"));
- if (!session) {
- graph = globalGraph;
- } else {
- /**
- * 分批创建session client，由于服务端groovy编译性能的限制
- */
- try {
- Thread.sleep((getTaskId()/10)*10000);
- } catch (Exception e) {
- // ...
- }
- graph = GdbGraphManager.instance().getGraph(taskConfig, session);
- }
+ if (!session) {
+ this.graph = globalGraph;
+ } else {
+ /**
+ * 分批创建session client，由于服务端groovy编译性能的限制
+ */
+ try {
+ Thread.sleep((getTaskId() / 10) * 10000);
+ } catch (final Exception e) {
+ // ...
+ }
+ this.graph = GdbGraphManager.instance().getGraph(this.taskConfig, session);
+ }
}
@Override
@@ -165,64 +168,69 @@ public class GdbWriter extends Writer {
* 注意：此方法每个 Task 都会执行一次。
* 最佳实践：如果 Task 中有需要进行数据同步之前的处理，可以在此处完成，如果没有必要则可以直接去掉。
*/
- super.prepare();
+ super.prepare();
}
@Override
- public void startWrite(RecordReceiver recordReceiver) {
+ public void startWrite(final RecordReceiver recordReceiver) {
/**
* 注意：此方法每个 Task 都会执行一次。
* 最佳实践：此处适当封装确保简洁清晰完成数据写入工作。
*/
- Record r;
- Future future = null;
- List> records = new ArrayList<>(batchRecords);
+ Record r;
+ Future future = null;
+ List> records = new ArrayList<>(this.batchRecords);
- while ((r = recordReceiver.getFromReader()) != null) {
- records.add(new Tuple2<>(r, mapper.apply(r)));
+ while ((r = recordReceiver.getFromReader()) != null) {
+ try {
+ records.add(new Tuple2<>(r, mapper.apply(r)));
+ } catch (final Exception ex) {
+ getTaskPluginCollector().collectDirtyRecord(r, ex);
+ continue;
+ }
- if (records.size() >= batchRecords) {
- wait4Submit(future);
+ if (records.size() >= this.batchRecords) {
+ wait4Submit(future);
- final List> batch = records;
- future = submitService.submit(() -> batchCommitRecords(batch));
- records = new ArrayList<>(batchRecords);
- }
- }
+ final List> batch = records;
+ future = this.submitService.submit(() -> batchCommitRecords(batch));
+ records = new ArrayList<>(this.batchRecords);
+ }
+ }
- wait4Submit(future);
- if (!records.isEmpty()) {
- final List> batch = records;
- future = submitService.submit(() -> batchCommitRecords(batch));
- wait4Submit(future);
- }
+ wait4Submit(future);
+ if (!records.isEmpty()) {
+ final List> batch = records;
+ future = this.submitService.submit(() -> batchCommitRecords(batch));
+ wait4Submit(future);
+ }
}
- private void wait4Submit(Future future) {
- if (future == null) {
- return;
- }
+ private void wait4Submit(final Future future) {
+ if (future == null) {
+ return;
+ }
- try {
- future.get();
- } catch (Exception e) {
- e.printStackTrace();
- }
+ try {
+ future.get();
+ } catch (final Exception e) {
+ e.printStackTrace();
+ }
}
private boolean batchCommitRecords(final List> records) {
- TaskPluginCollector collector = getTaskPluginCollector();
- try {
- List> errors = graph.add(records);
- errors.forEach(t -> collector.collectDirtyRecord(t.getFirst(), t.getSecond()));
- failed += errors.size();
- } catch (Exception e) {
- records.forEach(t -> collector.collectDirtyRecord(t.getFirst(), e));
- failed += records.size();
- }
+ final TaskPluginCollector collector = getTaskPluginCollector();
+ try {
+ final List> errors = this.graph.add(records);
+ errors.forEach(t -> collector.collectDirtyRecord(t.getFirst(), t.getSecond()));
+ this.failed += errors.size();
+ } catch (final Exception e) {
+ records.forEach(t -> collector.collectDirtyRecord(t.getFirst(), e));
+ this.failed += records.size();
+ }
- records.clear();
- return true;
+ records.clear();
+ return true;
}
@Override
@@ -231,7 +239,7 @@ public class GdbWriter extends Writer {
* 注意：此方法每个 Task 都会执行一次。
* 最佳实践：如果 Task 中有需要进行数据同步之后的后续处理，可以在此处完成。
*/
- log.info("Task done, dirty record count - {}", failed);
+ log.info("Task done, dirty record count - {}", this.failed);
}
@Override
@@ -241,9 +249,9 @@ public class GdbWriter extends Writer {
* 最佳实践：通常配合Task 中的 post() 方法一起完成 Task 的资源释放。
*/
if (session) {
- graph.close();
+ this.graph.close();
}
- submitService.shutdown();
+ this.submitService.shutdown();
}
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriterErrorCode.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriterErrorCode.java
index a6f506ef..e1c9080b 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriterErrorCode.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriterErrorCode.java
@@ -27,7 +27,6 @@ public enum GdbWriterErrorCode implements ErrorCode {
@Override
public String toString() {
- return String.format("Code:[%s], Description:[%s]. ", this.code,
- this.description);
+ return String.format("Code:[%s], Description:[%s]. ", this.code, this.description);
}
}
\ No newline at end of file
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/Key.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/Key.java
index f2e37005..afa58239 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/Key.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/Key.java
@@ -6,136 +6,164 @@ public final class Key {
* 此处声明插件用到的需要插件使用者提供的配置项
*/
- public final static String HOST = "host";
- public final static String PORT = "port";
+ public final static String HOST = "host";
+ public final static String PORT = "port";
public final static String USERNAME = "username";
- public static final String PASSWORD = "password";
+ public static final String PASSWORD = "password";
- /**
- * import type and mode
- */
- public static final String IMPORT_TYPE = "labelType";
- public static final String UPDATE_MODE = "writeMode";
+ /**
+ * import type and mode
+ */
+ public static final String IMPORT_TYPE = "labelType";
+ public static final String UPDATE_MODE = "writeMode";
- /**
- * label prefix issue
- */
- public static final String ID_TRANS_RULE = "idTransRule";
- public static final String SRC_ID_TRANS_RULE = "srcIdTransRule";
- public static final String DST_ID_TRANS_RULE = "dstIdTransRule";
+ /**
+ * label prefix issue
+ */
+ public static final String ID_TRANS_RULE = "idTransRule";
+ public static final String SRC_ID_TRANS_RULE = "srcIdTransRule";
+ public static final String DST_ID_TRANS_RULE = "dstIdTransRule";
- public static final String LABEL = "label";
- public static final String SRC_LABEL = "srcLabel";
- public static final String DST_LABEL = "dstLabel";
+ public static final String LABEL = "label";
+ public static final String SRC_LABEL = "srcLabel";
+ public static final String DST_LABEL = "dstLabel";
- public static final String MAPPING = "mapping";
+ public static final String MAPPING = "mapping";
- /**
- * column define in Gdb
- */
- public static final String COLUMN = "column";
- public static final String COLUMN_NAME = "name";
- public static final String COLUMN_VALUE = "value";
- public static final String COLUMN_TYPE = "type";
- public static final String COLUMN_NODE_TYPE = "columnType";
+ /**
+ * column define in Gdb
+ */
+ public static final String COLUMN = "column";
+ public static final String COLUMN_NAME = "name";
+ public static final String COLUMN_VALUE = "value";
+ public static final String COLUMN_TYPE = "type";
+ public static final String COLUMN_NODE_TYPE = "columnType";
- /**
- * Gdb Vertex/Edge elements
- */
- public static final String ID = "id";
- public static final String FROM = "from";
- public static final String TO = "to";
- public static final String PROPERTIES = "properties";
- public static final String PROP_KEY = "name";
- public static final String PROP_VALUE = "value";
- public static final String PROP_TYPE = "type";
+ /**
+ * Gdb Vertex/Edge elements
+ */
+ public static final String ID = "id";
+ public static final String FROM = "from";
+ public static final String TO = "to";
+ public static final String PROPERTIES = "properties";
+ public static final String PROP_KEY = "name";
+ public static final String PROP_VALUE = "value";
+ public static final String PROP_TYPE = "type";
- public static final String PROPERTIES_JSON_STR = "propertiesJsonStr";
- public static final String MAX_PROPERTIES_BATCH_NUM = "maxPropertiesBatchNumber";
+ public static final String PROPERTIES_JSON_STR = "propertiesJsonStr";
+ public static final String MAX_PROPERTIES_BATCH_NUM = "maxPropertiesBatchNumber";
- /**
- * session less client configure for connect pool
- */
- public static final String MAX_IN_PROCESS_PER_CONNECTION = "maxInProcessPerConnection";
- public static final String MAX_CONNECTION_POOL_SIZE = "maxConnectionPoolSize";
- public static final String MAX_SIMULTANEOUS_USAGE_PER_CONNECTION = "maxSimultaneousUsagePerConnection";
+ /**
+ * session less client configure for connect pool
+ */
+ public static final String MAX_IN_PROCESS_PER_CONNECTION = "maxInProcessPerConnection";
+ public static final String MAX_CONNECTION_POOL_SIZE = "maxConnectionPoolSize";
+ public static final String MAX_SIMULTANEOUS_USAGE_PER_CONNECTION = "maxSimultaneousUsagePerConnection";
- public static final String MAX_RECORDS_IN_BATCH = "maxRecordsInBatch";
- public static final String SESSION_STATE = "session";
+ public static final String MAX_RECORDS_IN_BATCH = "maxRecordsInBatch";
+ public static final String SESSION_STATE = "session";
- public static enum ImportType {
- /**
- * Import vertices
- */
- VERTEX,
- /**
- * Import edges
- */
- EDGE;
- }
-
- public static enum UpdateMode {
- /**
- * Insert new records, fail if exists
- */
- INSERT,
- /**
- * Skip this record if exists
- */
- SKIP,
- /**
- * Update property of this record if exists
- */
- MERGE;
- }
+ /**
+ * request length limit, include gdb element string length GDB字段长度限制配置，可分别配置各字段的限制，超过限制的记录会当脏数据处理
+ */
+ public static final String MAX_GDB_STRING_LENGTH = "maxStringLengthLimit";
+ public static final String MAX_GDB_ID_LENGTH = "maxIdStringLengthLimit";
+ public static final String MAX_GDB_LABEL_LENGTH = "maxLabelStringLengthLimit";
+ public static final String MAX_GDB_PROP_KEY_LENGTH = "maxPropKeyStringLengthLimit";
+ public static final String MAX_GDB_PROP_VALUE_LENGTH = "maxPropValueStringLengthLimit";
- public static enum ColumnType {
- /**
- * vertex or edge id
- */
- primaryKey,
+ public static final String MAX_GDB_REQUEST_LENGTH = "maxRequestLengthLimit";
- /**
- * vertex property
- */
- vertexProperty,
+ public static enum ImportType {
+ /**
+ * Import vertices
+ */
+ VERTEX,
+ /**
+ * Import edges
+ */
+ EDGE;
+ }
- /**
- * start vertex id of edge
- */
- srcPrimaryKey,
+ public static enum UpdateMode {
+ /**
+ * Insert new records, fail if exists
+ */
+ INSERT,
+ /**
+ * Skip this record if exists
+ */
+ SKIP,
+ /**
+ * Update property of this record if exists
+ */
+ MERGE;
+ }
- /**
- * end vertex id of edge
- */
- dstPrimaryKey,
+ public static enum ColumnType {
+ /**
+ * vertex or edge id
+ */
+ primaryKey,
- /**
- * edge property
- */
- edgeProperty,
+ /**
+ * vertex property
+ */
+ vertexProperty,
- /**
- * vertex json style property
- */
- vertexJsonProperty,
+ /**
+ * vertex setProperty
+ */
+ vertexSetProperty,
- /**
- * edge json style property
- */
- edgeJsonProperty
- }
+ /**
+ * start vertex id of edge
+ */
+ srcPrimaryKey,
- public static enum IdTransRule {
- /**
- * vertex or edge id with 'label' prefix
- */
- labelPrefix,
+ /**
+ * end vertex id of edge
+ */
+ dstPrimaryKey,
- /**
- * vertex or edge id raw
- */
- none
- }
+ /**
+ * edge property
+ */
+ edgeProperty,
+
+ /**
+ * vertex json style property
+ */
+ vertexJsonProperty,
+
+ /**
+ * edge json style property
+ */
+ edgeJsonProperty
+ }
+
+ public static enum IdTransRule {
+ /**
+ * vertex or edge id with 'label' prefix
+ */
+ labelPrefix,
+
+ /**
+ * vertex or edge id raw
+ */
+ none
+ }
+
+ public static enum PropertyType {
+ /**
+ * single Vertex Property
+ */
+ single,
+
+ /**
+ * set Vertex Property
+ */
+ set
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/client/GdbGraphManager.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/client/GdbGraphManager.java
index ac06013c..53668127 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/client/GdbGraphManager.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/client/GdbGraphManager.java
@@ -3,37 +3,37 @@
*/
package com.alibaba.datax.plugin.writer.gdbwriter.client;
+import java.util.ArrayList;
+import java.util.List;
+
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbGraph;
import com.alibaba.datax.plugin.writer.gdbwriter.model.ScriptGdbGraph;
-import java.util.ArrayList;
-import java.util.List;
-
/**
* @author jerrywang
*
*/
public class GdbGraphManager implements AutoCloseable {
- private static final GdbGraphManager instance = new GdbGraphManager();
-
- private List graphs = new ArrayList<>();
-
- public static GdbGraphManager instance() {
- return instance;
- }
+ private static final GdbGraphManager INSTANCE = new GdbGraphManager();
- public GdbGraph getGraph(Configuration config, boolean session) {
- GdbGraph graph = new ScriptGdbGraph(config, session);
- graphs.add(graph);
- return graph;
- }
+ private List graphs = new ArrayList<>();
- @Override
- public void close() {
- for(GdbGraph graph : graphs) {
- graph.close();
- }
- graphs.clear();
- }
+ public static GdbGraphManager instance() {
+ return INSTANCE;
+ }
+
+ public GdbGraph getGraph(final Configuration config, final boolean session) {
+ final GdbGraph graph = new ScriptGdbGraph(config, session);
+ this.graphs.add(graph);
+ return graph;
+ }
+
+ @Override
+ public void close() {
+ for (final GdbGraph graph : this.graphs) {
+ graph.close();
+ }
+ this.graphs.clear();
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/client/GdbWriterConfig.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/client/GdbWriterConfig.java
index 0266a010..dbc68b90 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/client/GdbWriterConfig.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/client/GdbWriterConfig.java
@@ -3,39 +3,43 @@
*/
package com.alibaba.datax.plugin.writer.gdbwriter.client;
+import static com.alibaba.datax.plugin.writer.gdbwriter.util.ConfigHelper.assertConfig;
+import static com.alibaba.datax.plugin.writer.gdbwriter.util.ConfigHelper.assertHasContent;
+
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.writer.gdbwriter.Key;
-import static com.alibaba.datax.plugin.writer.gdbwriter.util.ConfigHelper.*;
-
/**
* @author jerrywang
*
*/
public class GdbWriterConfig {
- public static final int DEFAULT_MAX_IN_PROCESS_PER_CONNECTION = 4;
- public static final int DEFAULT_MAX_CONNECTION_POOL_SIZE = 8;
- public static final int DEFAULT_MAX_SIMULTANEOUS_USAGE_PER_CONNECTION = 8;
- public static final int DEFAULT_BATCH_PROPERTY_NUM = 30;
- public static final int DEFAULT_RECORD_NUM_IN_BATCH = 16;
+ public static final int DEFAULT_MAX_IN_PROCESS_PER_CONNECTION = 4;
+ public static final int DEFAULT_MAX_CONNECTION_POOL_SIZE = 8;
+ public static final int DEFAULT_MAX_SIMULTANEOUS_USAGE_PER_CONNECTION = 8;
+ public static final int DEFAULT_BATCH_PROPERTY_NUM = 30;
+ public static final int DEFAULT_RECORD_NUM_IN_BATCH = 16;
- private Configuration config;
+ public static final int MAX_STRING_LENGTH = 10240;
+ public static final int MAX_REQUEST_LENGTH = 65535 - 1000;
- private GdbWriterConfig(Configuration config) {
- this.config = config;
+ private Configuration config;
- validate();
- }
+ private GdbWriterConfig(final Configuration config) {
+ this.config = config;
- private void validate() {
- assertHasContent(config, Key.HOST);
- assertConfig(Key.PORT, () -> config.getInt(Key.PORT) > 0);
+ validate();
+ }
- assertHasContent(config, Key.USERNAME);
- assertHasContent(config, Key.PASSWORD);
- }
-
- public static GdbWriterConfig of(Configuration config) {
- return new GdbWriterConfig(config);
- }
+ public static GdbWriterConfig of(final Configuration config) {
+ return new GdbWriterConfig(config);
+ }
+
+ private void validate() {
+ assertHasContent(this.config, Key.HOST);
+ assertConfig(Key.PORT, () -> this.config.getInt(Key.PORT) > 0);
+
+ assertHasContent(this.config, Key.USERNAME);
+ assertHasContent(this.config, Key.PASSWORD);
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/DefaultGdbMapper.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/DefaultGdbMapper.java
index f5957295..73a94cf5 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/DefaultGdbMapper.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/DefaultGdbMapper.java
@@ -3,6 +3,8 @@
*/
package com.alibaba.datax.plugin.writer.gdbwriter.mapping;
+import static com.alibaba.datax.plugin.writer.gdbwriter.Key.ImportType.VERTEX;
+
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -12,179 +14,191 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.alibaba.datax.common.element.Record;
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
+import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.writer.gdbwriter.Key;
import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbEdge;
import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbElement;
import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbVertex;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
-import static com.alibaba.datax.plugin.writer.gdbwriter.Key.ImportType.VERTEX;
-
/**
* @author jerrywang
*
*/
@Slf4j
public class DefaultGdbMapper implements GdbMapper {
- private static final Pattern STR_PATTERN = Pattern.compile("\\$\\{(\\d+)}");
- private static final Pattern NORMAL_PATTERN = Pattern.compile("^\\$\\{(\\d+)}$");
+ private static final Pattern STR_DOLLAR_PATTERN = Pattern.compile("\\$\\{(\\d+)}");
+ private static final Pattern NORMAL_DOLLAR_PATTERN = Pattern.compile("^\\$\\{(\\d+)}$");
- @Override
- public Function getMapper(MappingRule rule) {
- return r -> {
- GdbElement e = (rule.getImportType() == VERTEX) ? new GdbVertex() : new GdbEdge();
- forElement(rule).accept(r, e);
- return e;
+ private static final Pattern STR_NUM_PATTERN = Pattern.compile("#\\{(\\d+)}");
+ private static final Pattern NORMAL_NUM_PATTERN = Pattern.compile("^#\\{(\\d+)}$");
+
+ public DefaultGdbMapper() {}
+
+ public DefaultGdbMapper(final Configuration config) {
+ MapperConfig.getInstance().updateConfig(config);
+ }
+
+ private static BiConsumer forElement(final MappingRule rule) {
+ final boolean numPattern = rule.isNumPattern();
+ final List> properties = new ArrayList<>();
+ for (final MappingRule.PropertyMappingRule propRule : rule.getProperties()) {
+ final Function keyFunc = forStrColumn(numPattern, propRule.getKey());
+
+ if (propRule.getValueType() == ValueType.STRING) {
+ final Function valueFunc = forStrColumn(numPattern, propRule.getValue());
+ properties.add((r, e) -> {
+ e.addProperty(keyFunc.apply(r), valueFunc.apply(r), propRule.getPType());
+ });
+ } else {
+ final Function valueFunc =
+ forObjColumn(numPattern, propRule.getValue(), propRule.getValueType());
+ properties.add((r, e) -> {
+ e.addProperty(keyFunc.apply(r), valueFunc.apply(r), propRule.getPType());
+ });
+ }
+ }
+
+ if (rule.getPropertiesJsonStr() != null) {
+ final Function jsonFunc = forStrColumn(numPattern, rule.getPropertiesJsonStr());
+ properties.add((r, e) -> {
+ final String propertiesStr = jsonFunc.apply(r);
+ final JSONObject root = (JSONObject)JSONObject.parse(propertiesStr);
+ final JSONArray propertiesList = root.getJSONArray("properties");
+
+ for (final Object object : propertiesList) {
+ final JSONObject jsonObject = (JSONObject)object;
+ final String key = jsonObject.getString("k");
+ final String name = jsonObject.getString("v");
+ final String type = jsonObject.getString("t");
+ final String card = jsonObject.getString("c");
+
+ if (key == null || name == null) {
+ continue;
+ }
+ addToProperties(e, key, name, type, card);
+ }
+ });
+ }
+
+ final BiConsumer ret = (r, e) -> {
+ final String label = forStrColumn(numPattern, rule.getLabel()).apply(r);
+ String id = forStrColumn(numPattern, rule.getId()).apply(r);
+
+ if (rule.getImportType() == Key.ImportType.EDGE) {
+ final String to = forStrColumn(numPattern, rule.getTo()).apply(r);
+ final String from = forStrColumn(numPattern, rule.getFrom()).apply(r);
+ if (to == null || from == null) {
+ log.error("invalid record to: {} , from: {}", to, from);
+ throw new IllegalArgumentException("to or from missed in edge");
+ }
+ ((GdbEdge)e).setTo(to);
+ ((GdbEdge)e).setFrom(from);
+
+ // generate UUID for edge
+ if (id == null) {
+ id = UUID.randomUUID().toString();
+ }
+ }
+
+ if (id == null || label == null) {
+ log.error("invalid record id: {} , label: {}", id, label);
+ throw new IllegalArgumentException("id or label missed");
+ }
+
+ e.setId(id);
+ e.setLabel(label);
+
+ properties.forEach(p -> p.accept(r, e));
};
- }
+ return ret;
+ }
- private static BiConsumer forElement(MappingRule rule) {
- List> properties = new ArrayList<>();
- for (MappingRule.PropertyMappingRule propRule : rule.getProperties()) {
- Function keyFunc = forStrColumn(propRule.getKey());
+ private static Function forObjColumn(final boolean numPattern, final String rule, final ValueType type) {
+ final Pattern pattern = numPattern ? NORMAL_NUM_PATTERN : NORMAL_DOLLAR_PATTERN;
+ final Matcher m = pattern.matcher(rule);
+ if (m.matches()) {
+ final int index = Integer.valueOf(m.group(1));
+ return r -> type.applyColumn(r.getColumn(index));
+ } else {
+ return r -> type.fromStrFunc(rule);
+ }
+ }
- if (propRule.getValueType() == ValueType.STRING) {
- final Function valueFunc = forStrColumn(propRule.getValue());
- properties.add((r, e) -> {
- String k = keyFunc.apply(r);
- String v = valueFunc.apply(r);
- if (k != null && v != null) {
- e.getProperties().put(k, v);
- }
- });
- } else {
- final Function valueFunc = forObjColumn(propRule.getValue(), propRule.getValueType());
- properties.add((r, e) -> {
- String k = keyFunc.apply(r);
- Object v = valueFunc.apply(r);
- if (k != null && v != null) {
- e.getProperties().put(k, v);
- }
- });
- }
- }
+ private static Function forStrColumn(final boolean numPattern, final String rule) {
+ final List> list = new ArrayList<>();
+ final Pattern pattern = numPattern ? STR_NUM_PATTERN : STR_DOLLAR_PATTERN;
+ final Matcher m = pattern.matcher(rule);
+ int last = 0;
+ while (m.find()) {
+ final String index = m.group(1);
+ // as simple integer index.
+ final int i = Integer.parseInt(index);
- if (rule.getPropertiesJsonStr() != null) {
- Function jsonFunc = forStrColumn(rule.getPropertiesJsonStr());
- properties.add((r, e) -> {
- String propertiesStr = jsonFunc.apply(r);
- JSONObject root = (JSONObject)JSONObject.parse(propertiesStr);
- JSONArray propertiesList = root.getJSONArray("properties");
+ final int tmp = last;
+ final int start = m.start();
+ list.add((sb, record) -> {
+ sb.append(rule.subSequence(tmp, start));
+ if (record.getColumn(i) != null && record.getColumn(i).getByteSize() > 0) {
+ sb.append(record.getColumn(i).asString());
+ }
+ });
- for (Object object : propertiesList) {
- JSONObject jsonObject = (JSONObject)object;
- String key = jsonObject.getString("k");
- String name = jsonObject.getString("v");
- String type = jsonObject.getString("t");
+ last = m.end();
+ }
- if (key == null || name == null) {
- continue;
- }
- addToProperties(e, key, name, type);
- }
- });
- }
+ final int tmp = last;
+ list.add((sb, record) -> {
+ sb.append(rule.subSequence(tmp, rule.length()));
+ });
- BiConsumer ret = (r, e) -> {
- String label = forStrColumn(rule.getLabel()).apply(r);
- String id = forStrColumn(rule.getId()).apply(r);
+ return r -> {
+ final StringBuilder sb = new StringBuilder();
+ list.forEach(c -> c.accept(sb, r));
+ final String res = sb.toString();
+ return res.isEmpty() ? null : res;
+ };
+ }
- if (rule.getImportType() == Key.ImportType.EDGE) {
- String to = forStrColumn(rule.getTo()).apply(r);
- String from = forStrColumn(rule.getFrom()).apply(r);
- if (to == null || from == null) {
- log.error("invalid record to: {} , from: {}", to, from);
- throw new IllegalArgumentException("to or from missed in edge");
- }
- ((GdbEdge)e).setTo(to);
- ((GdbEdge)e).setFrom(from);
+ private static boolean addToProperties(final GdbElement e, final String key, final String value, final String type, final String card) {
+ final Object pValue;
+ final ValueType valueType = ValueType.fromShortName(type);
- // generate UUID for edge
- if (id == null) {
- id = UUID.randomUUID().toString();
- }
- }
+ if (valueType == ValueType.STRING) {
+ pValue = value;
+ } else if (valueType == ValueType.INT || valueType == ValueType.INTEGER) {
+ pValue = Integer.valueOf(value);
+ } else if (valueType == ValueType.LONG) {
+ pValue = Long.valueOf(value);
+ } else if (valueType == ValueType.DOUBLE) {
+ pValue = Double.valueOf(value);
+ } else if (valueType == ValueType.FLOAT) {
+ pValue = Float.valueOf(value);
+ } else if (valueType == ValueType.BOOLEAN) {
+ pValue = Boolean.valueOf(value);
+ } else {
+ log.error("invalid property key {}, value {}, type {}", key, value, type);
+ return false;
+ }
- if (id == null || label == null) {
- log.error("invalid record id: {} , label: {}", id, label);
- throw new IllegalArgumentException("id or label missed");
- }
+ // apply vertexSetProperty
+ if (Key.PropertyType.set.name().equals(card) && (e instanceof GdbVertex)) {
+ e.addProperty(key, pValue, Key.PropertyType.set);
+ } else {
+ e.addProperty(key, pValue);
+ }
+ return true;
+ }
- e.setId(id);
- e.setLabel(label);
-
- properties.forEach(p -> p.accept(r, e));
- };
- return ret;
- }
-
- static Function forObjColumn(String rule, ValueType type) {
- Matcher m = NORMAL_PATTERN.matcher(rule);
- if (m.matches()) {
- int index = Integer.valueOf(m.group(1));
- return r -> type.applyColumn(r.getColumn(index));
- } else {
- return r -> type.fromStrFunc(rule);
- }
- }
-
- static Function forStrColumn(String rule) {
- List> list = new ArrayList<>();
- Matcher m = STR_PATTERN.matcher(rule);
- int last = 0;
- while (m.find()) {
- String index = m.group(1);
- // as simple integer index.
- int i = Integer.parseInt(index);
-
- final int tmp = last;
- final int start = m.start();
- list.add((sb, record) -> {
- sb.append(rule.subSequence(tmp, start));
- if(record.getColumn(i) != null && record.getColumn(i).getByteSize() > 0) {
- sb.append(record.getColumn(i).asString());
- }
- });
-
- last = m.end();
- }
-
- final int tmp = last;
- list.add((sb, record) -> {
- sb.append(rule.subSequence(tmp, rule.length()));
- });
-
- return r -> {
- StringBuilder sb = new StringBuilder();
- list.forEach(c -> c.accept(sb, r));
- String res = sb.toString();
- return res.isEmpty() ? null : res;
- };
- }
-
- static boolean addToProperties(GdbElement e, String key, String value, String type) {
- ValueType valueType = ValueType.fromShortName(type);
-
- if(valueType == ValueType.STRING) {
- e.getProperties().put(key, value);
- } else if (valueType == ValueType.INT) {
- e.getProperties().put(key, Integer.valueOf(value));
- } else if (valueType == ValueType.LONG) {
- e.getProperties().put(key, Long.valueOf(value));
- } else if (valueType == ValueType.DOUBLE) {
- e.getProperties().put(key, Double.valueOf(value));
- } else if (valueType == ValueType.FLOAT) {
- e.getProperties().put(key, Float.valueOf(value));
- } else if (valueType == ValueType.BOOLEAN) {
- e.getProperties().put(key, Boolean.valueOf(value));
- } else {
- log.error("invalid property key {}, value {}, type {}", key, value, type);
- return false;
- }
-
- return true;
- }
+ @Override
+ public Function getMapper(final MappingRule rule) {
+ return r -> {
+ final GdbElement e = (rule.getImportType() == VERTEX) ? new GdbVertex() : new GdbEdge();
+ forElement(rule).accept(r, e);
+ return e;
+ };
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/GdbMapper.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/GdbMapper.java
index 3282f203..6a717a95 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/GdbMapper.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/GdbMapper.java
@@ -13,5 +13,5 @@ import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbElement;
*
*/
public interface GdbMapper {
- Function getMapper(MappingRule rule);
+ Function getMapper(MappingRule rule);
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MapperConfig.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MapperConfig.java
new file mode 100644
index 00000000..241cd31a
--- /dev/null
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MapperConfig.java
@@ -0,0 +1,68 @@
+/*
+ * (C) 2019-present Alibaba Group Holding Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
+ * License version 2 as published by the Free Software Foundation.
+ */
+package com.alibaba.datax.plugin.writer.gdbwriter.mapping;
+
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.writer.gdbwriter.Key;
+import com.alibaba.datax.plugin.writer.gdbwriter.client.GdbWriterConfig;
+
+/**
+ * @author : Liu Jianping
+ * @date : 2019/10/15
+ */
+
+public class MapperConfig {
+ private static MapperConfig instance = new MapperConfig();
+ private int maxIdLength;
+ private int maxLabelLength;
+ private int maxPropKeyLength;
+ private int maxPropValueLength;
+
+ private MapperConfig() {
+ this.maxIdLength = GdbWriterConfig.MAX_STRING_LENGTH;
+ this.maxLabelLength = GdbWriterConfig.MAX_STRING_LENGTH;
+ this.maxPropKeyLength = GdbWriterConfig.MAX_STRING_LENGTH;
+ this.maxPropValueLength = GdbWriterConfig.MAX_STRING_LENGTH;
+ }
+
+ public static MapperConfig getInstance() {
+ return instance;
+ }
+
+ public void updateConfig(final Configuration config) {
+ final int length = config.getInt(Key.MAX_GDB_STRING_LENGTH, GdbWriterConfig.MAX_STRING_LENGTH);
+
+ Integer sLength = config.getInt(Key.MAX_GDB_ID_LENGTH);
+ this.maxIdLength = sLength == null ? length : sLength;
+
+ sLength = config.getInt(Key.MAX_GDB_LABEL_LENGTH);
+ this.maxLabelLength = sLength == null ? length : sLength;
+
+ sLength = config.getInt(Key.MAX_GDB_PROP_KEY_LENGTH);
+ this.maxPropKeyLength = sLength == null ? length : sLength;
+
+ sLength = config.getInt(Key.MAX_GDB_PROP_VALUE_LENGTH);
+ this.maxPropValueLength = sLength == null ? length : sLength;
+ }
+
+ public int getMaxIdLength() {
+ return this.maxIdLength;
+ }
+
+ public int getMaxLabelLength() {
+ return this.maxLabelLength;
+ }
+
+ public int getMaxPropKeyLength() {
+ return this.maxPropKeyLength;
+ }
+
+ public int getMaxPropValueLength() {
+ return this.maxPropValueLength;
+ }
+
+}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRule.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRule.java
index c0c58d88..971fd6da 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRule.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRule.java
@@ -7,6 +7,7 @@ import java.util.ArrayList;
import java.util.List;
import com.alibaba.datax.plugin.writer.gdbwriter.Key.ImportType;
+import com.alibaba.datax.plugin.writer.gdbwriter.Key.PropertyType;
import lombok.Data;
@@ -16,26 +17,30 @@ import lombok.Data;
*/
@Data
public class MappingRule {
- private String id = null;
+ private String id = null;
- private String label = null;
-
- private ImportType importType = null;
-
- private String from = null;
+ private String label = null;
- private String to = null;
+ private ImportType importType = null;
- private List properties = new ArrayList<>();
+ private String from = null;
- private String propertiesJsonStr = null;
+ private String to = null;
- @Data
- public static class PropertyMappingRule {
- private String key = null;
-
- private String value = null;
-
- private ValueType valueType = null;
- }
+ private List properties = new ArrayList<>();
+
+ private String propertiesJsonStr = null;
+
+ private boolean numPattern = false;
+
+ @Data
+ public static class PropertyMappingRule {
+ private String key = null;
+
+ private String value = null;
+
+ private ValueType valueType = null;
+
+ private PropertyType pType = PropertyType.single;
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRuleFactory.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRuleFactory.java
index 0738ac17..3e3a2afe 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRuleFactory.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRuleFactory.java
@@ -3,18 +3,21 @@
*/
package com.alibaba.datax.plugin.writer.gdbwriter.mapping;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.writer.gdbwriter.GdbWriterErrorCode;
import com.alibaba.datax.plugin.writer.gdbwriter.Key;
-import com.alibaba.datax.plugin.writer.gdbwriter.Key.ImportType;
-import com.alibaba.datax.plugin.writer.gdbwriter.Key.IdTransRule;
import com.alibaba.datax.plugin.writer.gdbwriter.Key.ColumnType;
+import com.alibaba.datax.plugin.writer.gdbwriter.Key.IdTransRule;
+import com.alibaba.datax.plugin.writer.gdbwriter.Key.ImportType;
import com.alibaba.datax.plugin.writer.gdbwriter.mapping.MappingRule.PropertyMappingRule;
import com.alibaba.datax.plugin.writer.gdbwriter.util.ConfigHelper;
-import lombok.extern.slf4j.Slf4j;
-import java.util.List;
+import lombok.extern.slf4j.Slf4j;
/**
* @author jerrywang
@@ -22,66 +25,94 @@ import java.util.List;
*/
@Slf4j
public class MappingRuleFactory {
- private static final MappingRuleFactory instance = new MappingRuleFactory();
-
- public static final MappingRuleFactory getInstance() {
- return instance;
- }
+ private static final MappingRuleFactory instance = new MappingRuleFactory();
+ private static final Pattern STR_PATTERN = Pattern.compile("\\$\\{(\\d+)}");
+ private static final Pattern STR_NUM_PATTERN = Pattern.compile("#\\{(\\d+)}");
- @Deprecated
- public MappingRule create(Configuration config, ImportType type) {
- MappingRule rule = new MappingRule();
- rule.setId(config.getString(Key.ID));
- rule.setLabel(config.getString(Key.LABEL));
- if (type == ImportType.EDGE) {
- rule.setFrom(config.getString(Key.FROM));
- rule.setTo(config.getString(Key.TO));
- }
-
- rule.setImportType(type);
-
- List configurations = config.getListConfiguration(Key.PROPERTIES);
- if (configurations != null) {
- for (Configuration prop : config.getListConfiguration(Key.PROPERTIES)) {
- PropertyMappingRule propRule = new PropertyMappingRule();
- propRule.setKey(prop.getString(Key.PROP_KEY));
- propRule.setValue(prop.getString(Key.PROP_VALUE));
- propRule.setValueType(ValueType.fromShortName(prop.getString(Key.PROP_TYPE).toLowerCase()));
- rule.getProperties().add(propRule);
- }
- }
-
- String propertiesJsonStr = config.getString(Key.PROPERTIES_JSON_STR, null);
- if (propertiesJsonStr != null) {
- rule.setPropertiesJsonStr(propertiesJsonStr);
- }
-
- return rule;
- }
-
- public MappingRule createV2(Configuration config) {
- try {
- ImportType type = ImportType.valueOf(config.getString(Key.IMPORT_TYPE));
- return createV2(config, type);
- } catch (NullPointerException e) {
- throw DataXException.asDataXException(GdbWriterErrorCode.CONFIG_ITEM_MISS, Key.IMPORT_TYPE);
- } catch (IllegalArgumentException e) {
- throw DataXException.asDataXException(GdbWriterErrorCode.BAD_CONFIG_VALUE, Key.IMPORT_TYPE);
- }
+ public static MappingRuleFactory getInstance() {
+ return instance;
}
- public MappingRule createV2(Configuration config, ImportType type) {
- MappingRule rule = new MappingRule();
+ private static boolean isPattern(final String value, final MappingRule rule, final boolean checked) {
+ if (checked) {
+ return true;
+ }
- ConfigHelper.assertHasContent(config, Key.LABEL);
- rule.setLabel(config.getString(Key.LABEL));
- rule.setImportType(type);
+ if (value == null || value.isEmpty()) {
+ return false;
+ }
- IdTransRule srcTransRule = IdTransRule.none;
+ Matcher m = STR_PATTERN.matcher(value);
+ if (m.find()) {
+ rule.setNumPattern(false);
+ return true;
+ }
+
+ m = STR_NUM_PATTERN.matcher(value);
+ if (m.find()) {
+ rule.setNumPattern(true);
+ return true;
+ }
+
+ return false;
+ }
+
+ @Deprecated
+ public MappingRule create(final Configuration config, final ImportType type) {
+ final MappingRule rule = new MappingRule();
+ rule.setId(config.getString(Key.ID));
+ rule.setLabel(config.getString(Key.LABEL));
+ if (type == ImportType.EDGE) {
+ rule.setFrom(config.getString(Key.FROM));
+ rule.setTo(config.getString(Key.TO));
+ }
+
+ rule.setImportType(type);
+
+ final List configurations = config.getListConfiguration(Key.PROPERTIES);
+ if (configurations != null) {
+ for (final Configuration prop : config.getListConfiguration(Key.PROPERTIES)) {
+ final PropertyMappingRule propRule = new PropertyMappingRule();
+ propRule.setKey(prop.getString(Key.PROP_KEY));
+ propRule.setValue(prop.getString(Key.PROP_VALUE));
+ propRule.setValueType(ValueType.fromShortName(prop.getString(Key.PROP_TYPE).toLowerCase()));
+ rule.getProperties().add(propRule);
+ }
+ }
+
+ final String propertiesJsonStr = config.getString(Key.PROPERTIES_JSON_STR, null);
+ if (propertiesJsonStr != null) {
+ rule.setPropertiesJsonStr(propertiesJsonStr);
+ }
+
+ return rule;
+ }
+
+ public MappingRule createV2(final Configuration config) {
+ try {
+ final ImportType type = ImportType.valueOf(config.getString(Key.IMPORT_TYPE));
+ return createV2(config, type);
+ } catch (final NullPointerException e) {
+ throw DataXException.asDataXException(GdbWriterErrorCode.CONFIG_ITEM_MISS, Key.IMPORT_TYPE);
+ } catch (final IllegalArgumentException e) {
+ throw DataXException.asDataXException(GdbWriterErrorCode.BAD_CONFIG_VALUE, Key.IMPORT_TYPE);
+ }
+ }
+
+ public MappingRule createV2(final Configuration config, final ImportType type) {
+ final MappingRule rule = new MappingRule();
+ boolean patternChecked = false;
+
+ ConfigHelper.assertHasContent(config, Key.LABEL);
+ rule.setLabel(config.getString(Key.LABEL));
+ rule.setImportType(type);
+ patternChecked = isPattern(rule.getLabel(), rule, patternChecked);
+
+ IdTransRule srcTransRule = IdTransRule.none;
IdTransRule dstTransRule = IdTransRule.none;
if (type == ImportType.EDGE) {
- ConfigHelper.assertHasContent(config, Key.SRC_ID_TRANS_RULE);
- ConfigHelper.assertHasContent(config, Key.DST_ID_TRANS_RULE);
+ ConfigHelper.assertHasContent(config, Key.SRC_ID_TRANS_RULE);
+ ConfigHelper.assertHasContent(config, Key.DST_ID_TRANS_RULE);
srcTransRule = IdTransRule.valueOf(config.getString(Key.SRC_ID_TRANS_RULE));
dstTransRule = IdTransRule.valueOf(config.getString(Key.DST_ID_TRANS_RULE));
@@ -94,88 +125,96 @@ public class MappingRuleFactory {
ConfigHelper.assertHasContent(config, Key.DST_LABEL);
}
}
- ConfigHelper.assertHasContent(config, Key.ID_TRANS_RULE);
- IdTransRule transRule = IdTransRule.valueOf(config.getString(Key.ID_TRANS_RULE));
+ ConfigHelper.assertHasContent(config, Key.ID_TRANS_RULE);
+ final IdTransRule transRule = IdTransRule.valueOf(config.getString(Key.ID_TRANS_RULE));
- List configurationList = config.getListConfiguration(Key.COLUMN);
- ConfigHelper.assertConfig(Key.COLUMN, () -> (configurationList != null && !configurationList.isEmpty()));
- for (Configuration column : configurationList) {
- ConfigHelper.assertHasContent(column, Key.COLUMN_NAME);
- ConfigHelper.assertHasContent(column, Key.COLUMN_VALUE);
- ConfigHelper.assertHasContent(column, Key.COLUMN_TYPE);
- ConfigHelper.assertHasContent(column, Key.COLUMN_NODE_TYPE);
+ final List configurationList = config.getListConfiguration(Key.COLUMN);
+ ConfigHelper.assertConfig(Key.COLUMN, () -> (configurationList != null && !configurationList.isEmpty()));
+ for (final Configuration column : configurationList) {
+ ConfigHelper.assertHasContent(column, Key.COLUMN_NAME);
+ ConfigHelper.assertHasContent(column, Key.COLUMN_VALUE);
+ ConfigHelper.assertHasContent(column, Key.COLUMN_TYPE);
+ ConfigHelper.assertHasContent(column, Key.COLUMN_NODE_TYPE);
- String columnValue = column.getString(Key.COLUMN_VALUE);
- ColumnType columnType = ColumnType.valueOf(column.getString(Key.COLUMN_NODE_TYPE));
- if (columnValue == null || columnValue.isEmpty()) {
- // only allow edge empty id
- ConfigHelper.assertConfig("empty column value",
- () -> (type == ImportType.EDGE && columnType == ColumnType.primaryKey));
- }
+ final String columnValue = column.getString(Key.COLUMN_VALUE);
+ final ColumnType columnType = ColumnType.valueOf(column.getString(Key.COLUMN_NODE_TYPE));
+ if (columnValue == null || columnValue.isEmpty()) {
+ // only allow edge empty id
+ ConfigHelper.assertConfig("empty column value",
+ () -> (type == ImportType.EDGE && columnType == ColumnType.primaryKey));
+ }
+ patternChecked = isPattern(columnValue, rule, patternChecked);
- if (columnType == ColumnType.primaryKey) {
- ValueType propType = ValueType.fromShortName(column.getString(Key.COLUMN_TYPE));
- ConfigHelper.assertConfig("only string is allowed in primary key", () -> (propType == ValueType.STRING));
+ if (columnType == ColumnType.primaryKey) {
+ final ValueType propType = ValueType.fromShortName(column.getString(Key.COLUMN_TYPE));
+ ConfigHelper.assertConfig("only string is allowed in primary key",
+ () -> (propType == ValueType.STRING));
- if (transRule == IdTransRule.labelPrefix) {
- rule.setId(config.getString(Key.LABEL) + columnValue);
- } else {
- rule.setId(columnValue);
- }
- } else if (columnType == ColumnType.edgeJsonProperty || columnType == ColumnType.vertexJsonProperty) {
- // only support one json property in column
- ConfigHelper.assertConfig("multi JsonProperty", () -> (rule.getPropertiesJsonStr() == null));
-
- rule.setPropertiesJsonStr(columnValue);
- } else if (columnType == ColumnType.vertexProperty || columnType == ColumnType.edgeProperty) {
- PropertyMappingRule propertyMappingRule = new PropertyMappingRule();
-
- propertyMappingRule.setKey(column.getString(Key.COLUMN_NAME));
- propertyMappingRule.setValue(columnValue);
- ValueType propType = ValueType.fromShortName(column.getString(Key.COLUMN_TYPE));
- ConfigHelper.assertConfig("unsupported property type", () -> propType != null);
-
- propertyMappingRule.setValueType(propType);
- rule.getProperties().add(propertyMappingRule);
- } else if (columnType == ColumnType.srcPrimaryKey) {
- if (type != ImportType.EDGE) {
- continue;
- }
-
- ValueType propType = ValueType.fromShortName(column.getString(Key.COLUMN_TYPE));
- ConfigHelper.assertConfig("only string is allowed in primary key", () -> (propType == ValueType.STRING));
-
- if (srcTransRule == IdTransRule.labelPrefix) {
- rule.setFrom(config.getString(Key.SRC_LABEL) + columnValue);
+ if (transRule == IdTransRule.labelPrefix) {
+ rule.setId(config.getString(Key.LABEL) + columnValue);
} else {
- rule.setFrom(columnValue);
+ rule.setId(columnValue);
}
- } else if (columnType == ColumnType.dstPrimaryKey) {
+ } else if (columnType == ColumnType.edgeJsonProperty || columnType == ColumnType.vertexJsonProperty) {
+ // only support one json property in column
+ ConfigHelper.assertConfig("multi JsonProperty", () -> (rule.getPropertiesJsonStr() == null));
+
+ rule.setPropertiesJsonStr(columnValue);
+ } else if (columnType == ColumnType.vertexProperty || columnType == ColumnType.edgeProperty
+ || columnType == ColumnType.vertexSetProperty) {
+ final PropertyMappingRule propertyMappingRule = new PropertyMappingRule();
+
+ propertyMappingRule.setKey(column.getString(Key.COLUMN_NAME));
+ propertyMappingRule.setValue(columnValue);
+ final ValueType propType = ValueType.fromShortName(column.getString(Key.COLUMN_TYPE));
+ ConfigHelper.assertConfig("unsupported property type", () -> propType != null);
+
+ if (columnType == ColumnType.vertexSetProperty) {
+ propertyMappingRule.setPType(Key.PropertyType.set);
+ }
+ propertyMappingRule.setValueType(propType);
+ rule.getProperties().add(propertyMappingRule);
+ } else if (columnType == ColumnType.srcPrimaryKey) {
if (type != ImportType.EDGE) {
continue;
}
- ValueType propType = ValueType.fromShortName(column.getString(Key.COLUMN_TYPE));
- ConfigHelper.assertConfig("only string is allowed in primary key", () -> (propType == ValueType.STRING));
+ final ValueType propType = ValueType.fromShortName(column.getString(Key.COLUMN_TYPE));
+ ConfigHelper.assertConfig("only string is allowed in primary key",
+ () -> (propType == ValueType.STRING));
+
+ if (srcTransRule == IdTransRule.labelPrefix) {
+ rule.setFrom(config.getString(Key.SRC_LABEL) + columnValue);
+ } else {
+ rule.setFrom(columnValue);
+ }
+ } else if (columnType == ColumnType.dstPrimaryKey) {
+ if (type != ImportType.EDGE) {
+ continue;
+ }
+
+ final ValueType propType = ValueType.fromShortName(column.getString(Key.COLUMN_TYPE));
+ ConfigHelper.assertConfig("only string is allowed in primary key",
+ () -> (propType == ValueType.STRING));
if (dstTransRule == IdTransRule.labelPrefix) {
rule.setTo(config.getString(Key.DST_LABEL) + columnValue);
} else {
rule.setTo(columnValue);
}
- }
- }
+ }
+ }
- if (rule.getImportType() == ImportType.EDGE) {
- if (rule.getId() == null) {
- rule.setId("");
- log.info("edge id is missed, uuid be default");
- }
- ConfigHelper.assertConfig("to needed in edge", () -> (rule.getTo() != null));
- ConfigHelper.assertConfig("from needed in edge", () -> (rule.getFrom() != null));
- }
- ConfigHelper.assertConfig("id needed", () -> (rule.getId() != null));
+ if (rule.getImportType() == ImportType.EDGE) {
+ if (rule.getId() == null) {
+ rule.setId("");
+ log.info("edge id is missed, uuid be default");
+ }
+ ConfigHelper.assertConfig("to needed in edge", () -> (rule.getTo() != null));
+ ConfigHelper.assertConfig("from needed in edge", () -> (rule.getFrom() != null));
+ }
+ ConfigHelper.assertConfig("id needed", () -> (rule.getId() != null));
- return rule;
- }
+ return rule;
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/ValueType.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/ValueType.java
index 9ad8bd8d..969fda3b 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/ValueType.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/ValueType.java
@@ -8,6 +8,7 @@ import java.util.Map;
import java.util.function.Function;
import com.alibaba.datax.common.element.Column;
+
import lombok.extern.slf4j.Slf4j;
/**
@@ -16,56 +17,61 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
public enum ValueType {
- INT(Integer.class, "int", Column::asLong, Integer::valueOf),
- LONG(Long.class, "long", Column::asLong, Long::valueOf),
- DOUBLE(Double.class, "double", Column::asDouble, Double::valueOf),
- FLOAT(Float.class, "float", Column::asDouble, Float::valueOf),
- BOOLEAN(Boolean.class, "boolean", Column::asBoolean, Boolean::valueOf),
- STRING(String.class, "string", Column::asString, String::valueOf);
+ /**
+ * property value type
+ */
+ INT(Integer.class, "int", Column::asLong, Integer::valueOf),
+ INTEGER(Integer.class, "integer", Column::asLong, Integer::valueOf),
+ LONG(Long.class, "long", Column::asLong, Long::valueOf),
+ DOUBLE(Double.class, "double", Column::asDouble, Double::valueOf),
+ FLOAT(Float.class, "float", Column::asDouble, Float::valueOf),
+ BOOLEAN(Boolean.class, "boolean", Column::asBoolean, Boolean::valueOf),
+ STRING(String.class, "string", Column::asString, String::valueOf);
- private Class type = null;
- private String shortName = null;
- private Function columnFunc = null;
- private Function fromStrFunc = null;
+ private Class type = null;
+ private String shortName = null;
+ private Function columnFunc = null;
+ private Function fromStrFunc = null;
- private ValueType(Class type, String name, Function columnFunc, Function fromStrFunc) {
- this.type = type;
- this.shortName = name;
- this.columnFunc = columnFunc;
- this.fromStrFunc = fromStrFunc;
-
- ValueTypeHolder.shortName2type.put(name, this);
- }
-
- public static ValueType fromShortName(String name) {
- return ValueTypeHolder.shortName2type.get(name);
- }
+ private ValueType(final Class type, final String name, final Function columnFunc,
+ final Function fromStrFunc) {
+ this.type = type;
+ this.shortName = name;
+ this.columnFunc = columnFunc;
+ this.fromStrFunc = fromStrFunc;
- public Class type() {
- return this.type;
- }
-
- public String shortName() {
- return this.shortName;
- }
-
- public Object applyColumn(Column column) {
- try {
- if (column == null) {
- return null;
- }
- return columnFunc.apply(column);
- } catch (Exception e) {
- log.error("applyColumn error {}, column {}", e.toString(), column);
- throw e;
- }
- }
-
- public Object fromStrFunc(String str) {
- return fromStrFunc.apply(str);
- }
+ ValueTypeHolder.shortName2type.put(name, this);
+ }
- private static class ValueTypeHolder {
- private static Map shortName2type = new HashMap<>();
- }
+ public static ValueType fromShortName(final String name) {
+ return ValueTypeHolder.shortName2type.get(name);
+ }
+
+ public Class type() {
+ return this.type;
+ }
+
+ public String shortName() {
+ return this.shortName;
+ }
+
+ public Object applyColumn(final Column column) {
+ try {
+ if (column == null) {
+ return null;
+ }
+ return this.columnFunc.apply(column);
+ } catch (final Exception e) {
+ log.error("applyColumn error {}, column {}", e.toString(), column);
+ throw e;
+ }
+ }
+
+ public Object fromStrFunc(final String str) {
+ return this.fromStrFunc.apply(str);
+ }
+
+ private static class ValueTypeHolder {
+ private static Map shortName2type = new HashMap<>();
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/AbstractGdbGraph.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/AbstractGdbGraph.java
index 0c31c644..038663ac 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/AbstractGdbGraph.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/AbstractGdbGraph.java
@@ -3,20 +3,24 @@
*/
package com.alibaba.datax.plugin.writer.gdbwriter.model;
-import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.datax.plugin.writer.gdbwriter.Key;
-import com.alibaba.datax.plugin.writer.gdbwriter.client.GdbWriterConfig;
+import static com.alibaba.datax.plugin.writer.gdbwriter.client.GdbWriterConfig.DEFAULT_BATCH_PROPERTY_NUM;
+import static com.alibaba.datax.plugin.writer.gdbwriter.client.GdbWriterConfig.MAX_REQUEST_LENGTH;
+
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
-import lombok.extern.slf4j.Slf4j;
import org.apache.tinkerpop.gremlin.driver.Client;
import org.apache.tinkerpop.gremlin.driver.Cluster;
import org.apache.tinkerpop.gremlin.driver.RequestOptions;
import org.apache.tinkerpop.gremlin.driver.ResultSet;
import org.apache.tinkerpop.gremlin.driver.ser.Serializers;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.writer.gdbwriter.Key;
+import com.alibaba.datax.plugin.writer.gdbwriter.client.GdbWriterConfig;
+
+import lombok.extern.slf4j.Slf4j;
/**
* @author jerrywang
@@ -24,128 +28,124 @@ import java.util.concurrent.TimeUnit;
*/
@Slf4j
public abstract class AbstractGdbGraph implements GdbGraph {
- private final static int DEFAULT_TIMEOUT = 30000;
+ private final static int DEFAULT_TIMEOUT = 30000;
- protected Client client = null;
- protected Key.UpdateMode updateMode = Key.UpdateMode.INSERT;
- protected int propertiesBatchNum = GdbWriterConfig.DEFAULT_BATCH_PROPERTY_NUM;
- protected boolean session = false;
+ protected Client client = null;
+ protected Key.UpdateMode updateMode = Key.UpdateMode.INSERT;
+ protected int propertiesBatchNum = DEFAULT_BATCH_PROPERTY_NUM;
+ protected boolean session = false;
+ protected int maxRequestLength = GdbWriterConfig.MAX_REQUEST_LENGTH;
+ protected AbstractGdbGraph() {}
- protected AbstractGdbGraph() {}
+ protected AbstractGdbGraph(final Configuration config, final boolean session) {
+ initClient(config, session);
+ }
- protected AbstractGdbGraph(Configuration config, boolean session) {
- initClient(config, session);
- }
+ protected void initClient(final Configuration config, final boolean session) {
+ this.updateMode = Key.UpdateMode.valueOf(config.getString(Key.UPDATE_MODE, "INSERT"));
+ log.info("init graphdb client");
+ final String host = config.getString(Key.HOST);
+ final int port = config.getInt(Key.PORT);
+ final String username = config.getString(Key.USERNAME);
+ final String password = config.getString(Key.PASSWORD);
+ int maxDepthPerConnection =
+ config.getInt(Key.MAX_IN_PROCESS_PER_CONNECTION, GdbWriterConfig.DEFAULT_MAX_IN_PROCESS_PER_CONNECTION);
- protected void initClient(Configuration config, boolean session) {
- updateMode = Key.UpdateMode.valueOf(config.getString(Key.UPDATE_MODE, "INSERT"));
- log.info("init graphdb client");
- String host = config.getString(Key.HOST);
- int port = config.getInt(Key.PORT);
- String username = config.getString(Key.USERNAME);
- String password = config.getString(Key.PASSWORD);
- int maxDepthPerConnection = config.getInt(Key.MAX_IN_PROCESS_PER_CONNECTION,
- GdbWriterConfig.DEFAULT_MAX_IN_PROCESS_PER_CONNECTION);
+ int maxConnectionPoolSize =
+ config.getInt(Key.MAX_CONNECTION_POOL_SIZE, GdbWriterConfig.DEFAULT_MAX_CONNECTION_POOL_SIZE);
- int maxConnectionPoolSize = config.getInt(Key.MAX_CONNECTION_POOL_SIZE,
- GdbWriterConfig.DEFAULT_MAX_CONNECTION_POOL_SIZE);
+ int maxSimultaneousUsagePerConnection = config.getInt(Key.MAX_SIMULTANEOUS_USAGE_PER_CONNECTION,
+ GdbWriterConfig.DEFAULT_MAX_SIMULTANEOUS_USAGE_PER_CONNECTION);
- int maxSimultaneousUsagePerConnection = config.getInt(Key.MAX_SIMULTANEOUS_USAGE_PER_CONNECTION,
- GdbWriterConfig.DEFAULT_MAX_SIMULTANEOUS_USAGE_PER_CONNECTION);
+ this.session = session;
+ if (this.session) {
+ maxConnectionPoolSize = GdbWriterConfig.DEFAULT_MAX_CONNECTION_POOL_SIZE;
+ maxDepthPerConnection = GdbWriterConfig.DEFAULT_MAX_IN_PROCESS_PER_CONNECTION;
+ maxSimultaneousUsagePerConnection = GdbWriterConfig.DEFAULT_MAX_SIMULTANEOUS_USAGE_PER_CONNECTION;
+ }
- this.session = session;
- if (this.session) {
- maxConnectionPoolSize = GdbWriterConfig.DEFAULT_MAX_CONNECTION_POOL_SIZE;
- maxDepthPerConnection = GdbWriterConfig.DEFAULT_MAX_IN_PROCESS_PER_CONNECTION;
- maxSimultaneousUsagePerConnection = GdbWriterConfig.DEFAULT_MAX_SIMULTANEOUS_USAGE_PER_CONNECTION;
- }
+ try {
+ final Cluster cluster = Cluster.build(host).port(port).credentials(username, password)
+ .serializer(Serializers.GRAPHBINARY_V1D0).maxContentLength(1048576)
+ .maxInProcessPerConnection(maxDepthPerConnection).minInProcessPerConnection(0)
+ .maxConnectionPoolSize(maxConnectionPoolSize).minConnectionPoolSize(maxConnectionPoolSize)
+ .maxSimultaneousUsagePerConnection(maxSimultaneousUsagePerConnection).resultIterationBatchSize(64)
+ .create();
+ this.client = session ? cluster.connect(UUID.randomUUID().toString()).init() : cluster.connect().init();
+ warmClient(maxConnectionPoolSize * maxDepthPerConnection);
+ } catch (final RuntimeException e) {
+ log.error("Failed to connect to GDB {}:{}, due to {}", host, port, e);
+ throw e;
+ }
- try {
- Cluster cluster = Cluster.build(host).port(port).credentials(username, password)
- .serializer(Serializers.GRAPHBINARY_V1D0)
- .maxContentLength(1048576)
- .maxInProcessPerConnection(maxDepthPerConnection)
- .minInProcessPerConnection(0)
- .maxConnectionPoolSize(maxConnectionPoolSize)
- .minConnectionPoolSize(maxConnectionPoolSize)
- .maxSimultaneousUsagePerConnection(maxSimultaneousUsagePerConnection)
- .resultIterationBatchSize(64)
- .create();
- client = session ? cluster.connect(UUID.randomUUID().toString()).init() : cluster.connect().init();
- warmClient(maxConnectionPoolSize*maxDepthPerConnection);
- } catch (RuntimeException e) {
- log.error("Failed to connect to GDB {}:{}, due to {}", host, port, e);
- throw e;
- }
+ this.propertiesBatchNum = config.getInt(Key.MAX_PROPERTIES_BATCH_NUM, DEFAULT_BATCH_PROPERTY_NUM);
+ this.maxRequestLength = config.getInt(Key.MAX_GDB_REQUEST_LENGTH, MAX_REQUEST_LENGTH);
+ }
- propertiesBatchNum = config.getInt(Key.MAX_PROPERTIES_BATCH_NUM, GdbWriterConfig.DEFAULT_BATCH_PROPERTY_NUM);
- }
+ /**
+ * @param dsl
+ * @param parameters
+ */
+ protected void runInternal(final String dsl, final Map parameters) throws Exception {
+ final RequestOptions.Builder options = RequestOptions.build().timeout(DEFAULT_TIMEOUT);
+ if (parameters != null && !parameters.isEmpty()) {
+ parameters.forEach(options::addParameter);
+ }
+ final ResultSet results = this.client.submitAsync(dsl, options.create()).get(DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS);
+ results.all().get(DEFAULT_TIMEOUT + 1000, TimeUnit.MILLISECONDS);
+ }
- /**
- * @param dsl
- * @param parameters
- */
- protected void runInternal(String dsl, final Map parameters) throws Exception {
- RequestOptions.Builder options = RequestOptions.build().timeout(DEFAULT_TIMEOUT);
- if (parameters != null && !parameters.isEmpty()) {
- parameters.forEach(options::addParameter);
- }
+ void beginTx() {
+ if (!this.session) {
+ return;
+ }
- ResultSet results = client.submitAsync(dsl, options.create()).get(DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS);
- results.all().get(DEFAULT_TIMEOUT + 1000, TimeUnit.MILLISECONDS);
- }
+ final String dsl = "g.tx().open()";
+ this.client.submit(dsl).all().join();
+ }
- void beginTx() {
- if (!session) {
- return;
- }
+ void doCommit() {
+ if (!this.session) {
+ return;
+ }
- String dsl = "g.tx().open()";
- client.submit(dsl).all().join();
- }
+ try {
+ final String dsl = "g.tx().commit()";
+ this.client.submit(dsl).all().join();
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
- void doCommit() {
- if (!session) {
- return;
- }
+ void doRollback() {
+ if (!this.session) {
+ return;
+ }
- try {
- String dsl = "g.tx().commit()";
- client.submit(dsl).all().join();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
+ final String dsl = "g.tx().rollback()";
+ this.client.submit(dsl).all().join();
+ }
- void doRollback() {
- if (!session) {
- return;
- }
+ private void warmClient(final int num) {
+ try {
+ beginTx();
+ runInternal("g.V('test')", null);
+ doCommit();
+ log.info("warm graphdb client over");
+ } catch (final Exception e) {
+ log.error("warmClient error");
+ doRollback();
+ throw new RuntimeException(e);
+ }
+ }
- String dsl = "g.tx().rollback()";
- client.submit(dsl).all().join();
- }
-
- private void warmClient(int num) {
- try {
- beginTx();
- runInternal("g.V('test')", null);
- doCommit();
- log.info("warm graphdb client over");
- } catch (Exception e) {
- log.error("warmClient error");
- doRollback();
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void close() {
- if (client != null) {
- log.info("close graphdb client");
- client.close();
- }
- }
+ @Override
+ public void close() {
+ if (this.client != null) {
+ log.info("close graphdb client");
+ this.client.close();
+ }
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbEdge.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbEdge.java
index d42c9182..0bd42057 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbEdge.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbEdge.java
@@ -3,7 +3,8 @@
*/
package com.alibaba.datax.plugin.writer.gdbwriter.model;
-import lombok.Data;
+import com.alibaba.datax.plugin.writer.gdbwriter.mapping.MapperConfig;
+
import lombok.EqualsAndHashCode;
import lombok.ToString;
@@ -11,10 +12,33 @@ import lombok.ToString;
* @author jerrywang
*
*/
-@Data
@EqualsAndHashCode(callSuper = true)
@ToString(callSuper = true)
public class GdbEdge extends GdbElement {
- private String from = null;
- private String to = null;
+ private String from = null;
+ private String to = null;
+
+ public String getFrom() {
+ return this.from;
+ }
+
+ public void setFrom(final String from) {
+ final int maxIdLength = MapperConfig.getInstance().getMaxIdLength();
+ if (from.length() > maxIdLength) {
+ throw new IllegalArgumentException("from length over limit(" + maxIdLength + ")");
+ }
+ this.from = from;
+ }
+
+ public String getTo() {
+ return this.to;
+ }
+
+ public void setTo(final String to) {
+ final int maxIdLength = MapperConfig.getInstance().getMaxIdLength();
+ if (to.length() > maxIdLength) {
+ throw new IllegalArgumentException("to length over limit(" + maxIdLength + ")");
+ }
+ this.to = to;
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbElement.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbElement.java
index af3c7090..3d513a6a 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbElement.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbElement.java
@@ -3,18 +3,107 @@
*/
package com.alibaba.datax.plugin.writer.gdbwriter.model;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.LinkedList;
+import java.util.List;
-import lombok.Data;
+import com.alibaba.datax.plugin.writer.gdbwriter.Key.PropertyType;
+import com.alibaba.datax.plugin.writer.gdbwriter.mapping.MapperConfig;
/**
* @author jerrywang
*
*/
-@Data
public class GdbElement {
- String id = null;
- String label = null;
- Map properties = new HashMap<>();
+ private String id = null;
+ private String label = null;
+ private List properties = new LinkedList<>();
+
+ public String getId() {
+ return this.id;
+ }
+
+ public void setId(final String id) {
+ final int maxIdLength = MapperConfig.getInstance().getMaxIdLength();
+ if (id.length() > maxIdLength) {
+ throw new IllegalArgumentException("id length over limit(" + maxIdLength + ")");
+ }
+ this.id = id;
+ }
+
+ public String getLabel() {
+ return this.label;
+ }
+
+ public void setLabel(final String label) {
+ final int maxLabelLength = MapperConfig.getInstance().getMaxLabelLength();
+ if (label.length() > maxLabelLength) {
+ throw new IllegalArgumentException("label length over limit(" + maxLabelLength + ")");
+ }
+ this.label = label;
+ }
+
+ public List getProperties() {
+ return this.properties;
+ }
+
+ public void addProperty(final String propKey, final Object propValue, final PropertyType card) {
+ if (propKey == null || propValue == null) {
+ return;
+ }
+
+ final int maxPropKeyLength = MapperConfig.getInstance().getMaxPropKeyLength();
+ if (propKey.length() > maxPropKeyLength) {
+ throw new IllegalArgumentException("property key length over limit(" + maxPropKeyLength + ")");
+ }
+ if (propValue instanceof String) {
+ final int maxPropValueLength = MapperConfig.getInstance().getMaxPropValueLength();
+ if (((String)propValue).length() > maxPropKeyLength) {
+ throw new IllegalArgumentException("property value length over limit(" + maxPropValueLength + ")");
+ }
+ }
+
+ this.properties.add(new GdbProperty(propKey, propValue, card));
+ }
+
+ public void addProperty(final String propKey, final Object propValue) {
+ addProperty(propKey, propValue, PropertyType.single);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuffer sb = new StringBuffer(this.id + "[" + this.label + "]{");
+ this.properties.forEach(n -> {
+ sb.append(n.cardinality.name());
+ sb.append("[");
+ sb.append(n.key);
+ sb.append(" - ");
+ sb.append(String.valueOf(n.value));
+ sb.append("]");
+ });
+ return sb.toString();
+ }
+
+ public static class GdbProperty {
+ private String key;
+ private Object value;
+ private PropertyType cardinality;
+
+ private GdbProperty(final String key, final Object value, final PropertyType card) {
+ this.key = key;
+ this.value = value;
+ this.cardinality = card;
+ }
+
+ public PropertyType getCardinality() {
+ return this.cardinality;
+ }
+
+ public String getKey() {
+ return this.key;
+ }
+
+ public Object getValue() {
+ return this.value;
+ }
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbGraph.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbGraph.java
index 5b98c502..5d9b4508 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbGraph.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbGraph.java
@@ -3,18 +3,19 @@
*/
package com.alibaba.datax.plugin.writer.gdbwriter.model;
-import com.alibaba.datax.common.element.Record;
-import groovy.lang.Tuple2;
-
import java.util.List;
+import com.alibaba.datax.common.element.Record;
+
+import groovy.lang.Tuple2;
+
/**
* @author jerrywang
*
*/
public interface GdbGraph extends AutoCloseable {
- List> add(List> records);
+ List> add(List> records);
- @Override
- void close();
+ @Override
+ void close();
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/ScriptGdbGraph.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/ScriptGdbGraph.java
index 7f898431..9ecee8ab 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/ScriptGdbGraph.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/ScriptGdbGraph.java
@@ -3,15 +3,17 @@
*/
package com.alibaba.datax.plugin.writer.gdbwriter.model;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.common.util.Configuration;
-
import com.alibaba.datax.plugin.writer.gdbwriter.Key;
import com.alibaba.datax.plugin.writer.gdbwriter.util.GdbDuplicateIdException;
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
+
import groovy.lang.Tuple2;
import lombok.extern.slf4j.Slf4j;
@@ -21,176 +23,198 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
public class ScriptGdbGraph extends AbstractGdbGraph {
- private static final String VAR_PREFIX = "GDB___";
- private static final String VAR_ID = VAR_PREFIX + "id";
- private static final String VAR_LABEL = VAR_PREFIX + "label";
- private static final String VAR_FROM = VAR_PREFIX + "from";
- private static final String VAR_TO = VAR_PREFIX + "to";
- private static final String VAR_PROP_KEY = VAR_PREFIX + "PK";
- private static final String VAR_PROP_VALUE = VAR_PREFIX + "PV";
- private static final String ADD_V_START = "g.addV(" + VAR_LABEL + ").property(id, " + VAR_ID + ")";
- private static final String ADD_E_START = "g.addE(" + VAR_LABEL + ").property(id, " + VAR_ID + ").from(V("
- + VAR_FROM + ")).to(V(" + VAR_TO + "))";
+ private static final String VAR_PREFIX = "GDB___";
+ private static final String VAR_ID = VAR_PREFIX + "id";
+ private static final String VAR_LABEL = VAR_PREFIX + "label";
+ private static final String VAR_FROM = VAR_PREFIX + "from";
+ private static final String VAR_TO = VAR_PREFIX + "to";
+ private static final String VAR_PROP_KEY = VAR_PREFIX + "PK";
+ private static final String VAR_PROP_VALUE = VAR_PREFIX + "PV";
+ private static final String ADD_V_START = "g.addV(" + VAR_LABEL + ").property(id, " + VAR_ID + ")";
+ private static final String ADD_E_START =
+ "g.addE(" + VAR_LABEL + ").property(id, " + VAR_ID + ").from(V(" + VAR_FROM + ")).to(V(" + VAR_TO + "))";
- private static final String UPDATE_V_START = "g.V("+VAR_ID+")";
- private static final String UPDATE_E_START = "g.E("+VAR_ID+")";
+ private static final String UPDATE_V_START = "g.V(" + VAR_ID + ")";
+ private static final String UPDATE_E_START = "g.E(" + VAR_ID + ")";
- private Cache propertyCache;
- private Random random;
+ private Random random;
- public ScriptGdbGraph() {
- propertyCache = Caffeine.newBuilder().maximumSize(1024).build();
- random = new Random();
- }
+ public ScriptGdbGraph() {
+ this.random = new Random();
+ }
- public ScriptGdbGraph(Configuration config, boolean session) {
- super(config, session);
+ public ScriptGdbGraph(final Configuration config, final boolean session) {
+ super(config, session);
- propertyCache = Caffeine.newBuilder().maximumSize(1024).build();
- random = new Random();
+ this.random = new Random();
+ log.info("Init as ScriptGdbGraph.");
+ }
- log.info("Init as ScriptGdbGraph.");
- }
+ /**
+ * Apply list of {@link GdbElement} to GDB, return the failed records
+ *
+ * @param records
+ * list of element to apply
+ * @return
+ */
+ @Override
+ public List> add(final List> records) {
+ final List> errors = new ArrayList<>();
+ try {
+ beginTx();
+ for (final Tuple2 elementTuple2 : records) {
+ try {
+ addInternal(elementTuple2.getSecond());
+ } catch (final Exception e) {
+ errors.add(new Tuple2<>(elementTuple2.getFirst(), e));
+ }
+ }
+ doCommit();
+ } catch (final Exception ex) {
+ doRollback();
+ throw new RuntimeException(ex);
+ }
+ return errors;
+ }
- /**
- * Apply list of {@link GdbElement} to GDB, return the failed records
- * @param records list of element to apply
- * @return
- */
- @Override
- public List> add(List> records) {
- List> errors = new ArrayList<>();
- try {
- beginTx();
- for (Tuple2 elementTuple2 : records) {
- try {
- addInternal(elementTuple2.getSecond());
- } catch (Exception e) {
- errors.add(new Tuple2<>(elementTuple2.getFirst(), e));
- }
- }
- doCommit();
- } catch (Exception ex) {
- doRollback();
- throw new RuntimeException(ex);
- }
- return errors;
- }
+ private void addInternal(final GdbElement element) {
+ try {
+ addInternal(element, false);
+ } catch (final GdbDuplicateIdException e) {
+ if (this.updateMode == Key.UpdateMode.SKIP) {
+ log.debug("Skip duplicate id {}", element.getId());
+ } else if (this.updateMode == Key.UpdateMode.INSERT) {
+ throw new RuntimeException(e);
+ } else if (this.updateMode == Key.UpdateMode.MERGE) {
+ if (element.getProperties().isEmpty()) {
+ return;
+ }
- private void addInternal(GdbElement element) {
- try {
- addInternal(element, false);
- } catch (GdbDuplicateIdException e) {
- if (updateMode == Key.UpdateMode.SKIP) {
- log.debug("Skip duplicate id {}", element.getId());
- } else if (updateMode == Key.UpdateMode.INSERT) {
- throw new RuntimeException(e);
- } else if (updateMode == Key.UpdateMode.MERGE) {
- if (element.getProperties().isEmpty()) {
- return;
- }
+ try {
+ addInternal(element, true);
+ } catch (final GdbDuplicateIdException e1) {
+ log.error("duplicate id {} while update...", element.getId());
+ throw new RuntimeException(e1);
+ }
+ }
+ }
+ }
- try {
- addInternal(element, true);
- } catch (GdbDuplicateIdException e1) {
- log.error("duplicate id {} while update...", element.getId());
- throw new RuntimeException(e1);
- }
- }
- }
- }
+ private void addInternal(final GdbElement element, final boolean update) throws GdbDuplicateIdException {
+ boolean firstAdd = !update;
+ final boolean isVertex = (element instanceof GdbVertex);
+ final List params = element.getProperties();
+ final List subParams = new ArrayList<>(this.propertiesBatchNum);
- private void addInternal(GdbElement element, boolean update) throws GdbDuplicateIdException {
- Map params = element.getProperties();
- Map subParams = new HashMap<>(propertiesBatchNum);
- boolean firstAdd = !update;
- boolean isVertex = (element instanceof GdbVertex);
+ final int idLength = element.getId().length();
+ int attachLength = element.getLabel().length();
+ if (element instanceof GdbEdge) {
+ attachLength += ((GdbEdge)element).getFrom().length();
+ attachLength += ((GdbEdge)element).getTo().length();
+ }
- for (Map.Entry entry : params.entrySet()) {
- subParams.put(entry.getKey(), entry.getValue());
- if (subParams.size() >= propertiesBatchNum) {
- setGraphDbElement(element, subParams, isVertex, firstAdd);
- firstAdd = false;
- subParams.clear();
- }
- }
- if (!subParams.isEmpty() || firstAdd) {
- setGraphDbElement(element, subParams, isVertex, firstAdd);
- }
- }
+ int requestLength = idLength;
+ for (final GdbElement.GdbProperty entry : params) {
+ final String propKey = entry.getKey();
+ final Object propValue = entry.getValue();
- private Tuple2> buildDsl(GdbElement element,
- Map properties,
- boolean isVertex, boolean firstAdd) {
- Map params = new HashMap<>();
+ int appendLength = propKey.length();
+ if (propValue instanceof String) {
+ appendLength += ((String)propValue).length();
+ }
- String dslPropertyPart = propertyCache.get(properties.size(), keys -> {
- final StringBuilder sb = new StringBuilder();
- for (int i = 0; i < keys; i++) {
- sb.append(".property(").append(VAR_PROP_KEY).append(i)
- .append(", ").append(VAR_PROP_VALUE).append(i).append(")");
- }
- return sb.toString();
- });
+ if (checkSplitDsl(firstAdd, requestLength, attachLength, appendLength, subParams.size())) {
+ setGraphDbElement(element, subParams, isVertex, firstAdd);
+ firstAdd = false;
+ subParams.clear();
+ requestLength = idLength;
+ }
- String dsl;
- if (isVertex) {
- dsl = (firstAdd ? ADD_V_START : UPDATE_V_START) + dslPropertyPart;
- } else {
- dsl = (firstAdd ? ADD_E_START : UPDATE_E_START) + dslPropertyPart;
- if (firstAdd) {
- params.put(VAR_FROM, ((GdbEdge)element).getFrom());
- params.put(VAR_TO, ((GdbEdge)element).getTo());
- }
- }
+ requestLength += appendLength;
+ subParams.add(entry);
+ }
+ if (!subParams.isEmpty() || firstAdd) {
+ checkSplitDsl(firstAdd, requestLength, attachLength, 0, 0);
+ setGraphDbElement(element, subParams, isVertex, firstAdd);
+ }
+ }
- int index = 0;
- for (Map.Entry entry : properties.entrySet()) {
- params.put(VAR_PROP_KEY+index, entry.getKey());
- params.put(VAR_PROP_VALUE+index, entry.getValue());
- index++;
- }
+ private boolean checkSplitDsl(final boolean firstAdd, final int requestLength, final int attachLength, final int appendLength,
+ final int propNum) {
+ final int length = firstAdd ? requestLength + attachLength : requestLength;
+ if (length > this.maxRequestLength) {
+ throw new IllegalArgumentException("request length over limit(" + this.maxRequestLength + ")");
+ }
+ return length + appendLength > this.maxRequestLength || propNum >= this.propertiesBatchNum;
+ }
- if (firstAdd) {
- params.put(VAR_LABEL, element.getLabel());
- }
- params.put(VAR_ID, element.getId());
+ private Tuple2> buildDsl(final GdbElement element, final List properties,
+ final boolean isVertex, final boolean firstAdd) {
+ final Map params = new HashMap<>();
+ final StringBuilder sb = new StringBuilder();
+ if (isVertex) {
+ sb.append(firstAdd ? ADD_V_START : UPDATE_V_START);
+ } else {
+ sb.append(firstAdd ? ADD_E_START : UPDATE_E_START);
+ }
- return new Tuple2<>(dsl, params);
- }
+ for (int i = 0; i < properties.size(); i++) {
+ final GdbElement.GdbProperty prop = properties.get(i);
- private void setGraphDbElement(GdbElement element, Map properties,
- boolean isVertex, boolean firstAdd) throws GdbDuplicateIdException {
- int retry = 10;
- int idleTime = random.nextInt(10) + 10;
- Tuple2> elementDsl = buildDsl(element, properties, isVertex, firstAdd);
+ sb.append(".property(");
+ if (prop.getCardinality() == Key.PropertyType.set) {
+ sb.append("set, ");
+ }
+ sb.append(VAR_PROP_KEY).append(i).append(", ").append(VAR_PROP_VALUE).append(i).append(")");
- while (retry > 0) {
- try {
- runInternal(elementDsl.getFirst(), elementDsl.getSecond());
- log.debug("AddElement {}", element.getId());
- return;
- } catch (Exception e) {
- String cause = e.getCause() == null ? "" : e.getCause().toString();
- if (cause.contains("rejected from")) {
- retry--;
- try {
- Thread.sleep(idleTime);
- } catch (InterruptedException e1) {
- // ...
- }
- idleTime = Math.min(idleTime * 2, 2000);
- continue;
- } else if (firstAdd && cause.contains("GraphDB id exists")) {
- throw new GdbDuplicateIdException(e);
- }
- log.error("Add Failed id {}, dsl {}, params {}, e {}", element.getId(),
- elementDsl.getFirst(), elementDsl.getSecond(), e);
- throw new RuntimeException(e);
- }
- }
- log.error("Add Failed id {}, dsl {}, params {}", element.getId(),
- elementDsl.getFirst(), elementDsl.getSecond());
- throw new RuntimeException("failed to queue new element to server");
- }
+ params.put(VAR_PROP_KEY + i, prop.getKey());
+ params.put(VAR_PROP_VALUE + i, prop.getValue());
+ }
+
+ if (firstAdd) {
+ params.put(VAR_LABEL, element.getLabel());
+ if (!isVertex) {
+ params.put(VAR_FROM, ((GdbEdge)element).getFrom());
+ params.put(VAR_TO, ((GdbEdge)element).getTo());
+ }
+ }
+ params.put(VAR_ID, element.getId());
+
+ return new Tuple2<>(sb.toString(), params);
+ }
+
+ private void setGraphDbElement(final GdbElement element, final List properties, final boolean isVertex,
+ final boolean firstAdd) throws GdbDuplicateIdException {
+ int retry = 10;
+ int idleTime = this.random.nextInt(10) + 10;
+ final Tuple2> elementDsl = buildDsl(element, properties, isVertex, firstAdd);
+
+ while (retry > 0) {
+ try {
+ runInternal(elementDsl.getFirst(), elementDsl.getSecond());
+ log.debug("AddElement {}", element.getId());
+ return;
+ } catch (final Exception e) {
+ final String cause = e.getCause() == null ? "" : e.getCause().toString();
+ if (cause.contains("rejected from") || cause.contains("Timeout waiting to lock key")) {
+ retry--;
+ try {
+ Thread.sleep(idleTime);
+ } catch (final InterruptedException e1) {
+ // ...
+ }
+ idleTime = Math.min(idleTime * 2, 2000);
+ continue;
+ } else if (firstAdd && cause.contains("GraphDB id exists")) {
+ throw new GdbDuplicateIdException(e);
+ }
+ log.error("Add Failed id {}, dsl {}, params {}, e {}", element.getId(), elementDsl.getFirst(),
+ elementDsl.getSecond(), e);
+ throw new RuntimeException(e);
+ }
+ }
+ log.error("Add Failed id {}, dsl {}, params {}", element.getId(), elementDsl.getFirst(),
+ elementDsl.getSecond());
+ throw new RuntimeException("failed to queue new element to server");
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/ConfigHelper.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/ConfigHelper.java
index 77175197..178b5e7c 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/ConfigHelper.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/ConfigHelper.java
@@ -7,53 +7,57 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
+
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.writer.gdbwriter.GdbWriterErrorCode;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
-import org.apache.commons.lang3.StringUtils;
-
/**
* @author jerrywang
*
*/
public interface ConfigHelper {
- static void assertConfig(String key, Supplier f) {
- if (!f.get()) {
- throw DataXException.asDataXException(GdbWriterErrorCode.BAD_CONFIG_VALUE, key);
- }
- }
+ static void assertConfig(final String key, final Supplier f) {
+ if (!f.get()) {
+ throw DataXException.asDataXException(GdbWriterErrorCode.BAD_CONFIG_VALUE, key);
+ }
+ }
- static void assertHasContent(Configuration config, String key) {
- assertConfig(key, () -> StringUtils.isNotBlank(config.getString(key)));
- }
+ static void assertHasContent(final Configuration config, final String key) {
+ assertConfig(key, () -> StringUtils.isNotBlank(config.getString(key)));
+ }
- /**
- * NOTE: {@code Configuration::get(String, Class)} doesn't work.
- *
- * @param conf Configuration
- * @param key key path to configuration
- * @param cls Class of result type
- * @return the target configuration object of type T
- */
- static T getConfig(Configuration conf, String key, Class cls) {
- JSONObject j = (JSONObject) conf.get(key);
- return JSON.toJavaObject(j, cls);
- }
-
- /**
- * Create a configuration from the specified file on the classpath.
- *
- * @param name file name
- * @return Configuration instance.
- */
- static Configuration fromClasspath(String name) {
- try (InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(name)) {
- return Configuration.from(is);
- } catch (IOException e) {
- throw new IllegalArgumentException("File not found: " + name);
- }
- }
+ /**
+ * NOTE: {@code Configuration::get(String, Class)} doesn't work.
+ *
+ * @param conf
+ * Configuration
+ * @param key
+ * key path to configuration
+ * @param cls
+ * Class of result type
+ * @return the target configuration object of type T
+ */
+ static T getConfig(final Configuration conf, final String key, final Class cls) {
+ final JSONObject j = (JSONObject)conf.get(key);
+ return JSON.toJavaObject(j, cls);
+ }
+
+ /**
+ * Create a configuration from the specified file on the classpath.
+ *
+ * @param name
+ * file name
+ * @return Configuration instance.
+ */
+ static Configuration fromClasspath(final String name) {
+ try (final InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(name)) {
+ return Configuration.from(is);
+ } catch (final IOException e) {
+ throw new IllegalArgumentException("File not found: " + name);
+ }
+ }
}
diff --git a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/GdbDuplicateIdException.java b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/GdbDuplicateIdException.java
index e531d51b..dba641b0 100644
--- a/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/GdbDuplicateIdException.java
+++ b/gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/GdbDuplicateIdException.java
@@ -1,9 +1,8 @@
/*
- * (C) 2019-present Alibaba Group Holding Limited.
+ * (C) 2019-present Alibaba Group Holding Limited.
*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
+ * License version 2 as published by the Free Software Foundation.
*/
package com.alibaba.datax.plugin.writer.gdbwriter.util;
@@ -13,11 +12,11 @@ package com.alibaba.datax.plugin.writer.gdbwriter.util;
*/
public class GdbDuplicateIdException extends Exception {
- public GdbDuplicateIdException(Exception e) {
- super(e);
- }
+ public GdbDuplicateIdException(Exception e) {
+ super(e);
+ }
- public GdbDuplicateIdException() {
- super();
- }
+ public GdbDuplicateIdException() {
+ super();
+ }
}
diff --git a/hbase11xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java b/hbase11xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java
index 2aacdddf..5309d1d9 100644
--- a/hbase11xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java
+++ b/hbase11xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xsqlreader/HbaseSQLHelper.java
@@ -34,6 +34,14 @@ import java.util.Map;
public class HbaseSQLHelper {
private static final Logger LOG = LoggerFactory.getLogger(HbaseSQLHelper.class);
+ static {
+ try {
+ Class.forName("org.apache.phoenix.jdbc.PhoenixDriver");
+ } catch (Throwable t) {
+ throw new RuntimeException("faild load org.apache.phoenix.jdbc.PhoenixDriver", t);
+ }
+ }
+
public static org.apache.hadoop.conf.Configuration generatePhoenixConf(HbaseSQLReaderConfig readerConfig) {
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
diff --git a/hbase11xwriter/doc/hbase11xwriter.md b/hbase11xwriter/doc/hbase11xwriter.md
index ff20abe9..969f2e47 100644
--- a/hbase11xwriter/doc/hbase11xwriter.md
+++ b/hbase11xwriter/doc/hbase11xwriter.md
@@ -203,19 +203,20 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba
* 描述：要写入的hbase字段。index：指定该列对应reader端column的索引，从0开始；name：指定hbase表中的列，必须为 列族:列名 的格式；type：指定写入数据类型，用于转换HBase byte[]。配置格式如下：
```
-"column": [
- {
- "index":1,
- "name": "cf1:q1",
- "type": "string"
- },
- {
- "index":2,
- "name": "cf1:q2",
- "type": "string"
- }
- ］
-
+
+ "column": [
+ {
+ "index":1,
+ "name": "cf1:q1",
+ "type": "string"
+ },
+ {
+ "index":2,
+ "name": "cf1:q2",
+ "type": "string"
+ }
+ ］
+
```
* 必选：是
@@ -227,17 +228,17 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba
* 描述：要写入的hbase的rowkey列。index：指定该列对应reader端column的索引，从0开始，若为常量index为－1；type：指定写入数据类型，用于转换HBase byte[]；value：配置常量，常作为多个字段的拼接符。hbasewriter会将rowkeyColumn中所有列按照配置顺序进行拼接作为写入hbase的rowkey，不能全为常量。配置格式如下：
```
-"rowkeyColumn": [
- {
- "index":0,
- "type":"string"
- },
- {
- "index":-1,
- "type":"string",
- "value":"_"
- }
- ]
+ "rowkeyColumn": [
+ {
+ "index":0,
+ "type":"string"
+ },
+ {
+ "index":-1,
+ "type":"string",
+ "value":"_"
+ }
+ ]
```
@@ -250,19 +251,19 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba
* 描述：指定写入hbase的时间戳。支持：当前时间、指定时间列，指定时间，三者选一。若不配置表示用当前时间。index：指定对应reader端column的索引，从0开始，需保证能转换为long,若是Date类型，会尝试用yyyy-MM-dd HH:mm:ss和yyyy-MM-dd HH:mm:ss SSS去解析；若为指定时间index为－1；value：指定时间的值,long值。配置格式如下：
```
-"versionColumn":{
- "index":1
-}
+ "versionColumn":{
+ "index":1
+ }
```
或者
```
-"versionColumn":{
- "index":－1,
- "value":123456789
-}
+ "versionColumn":{
+ "index":－1,
+ "value":123456789
+ }
```
diff --git a/hbase20xsqlreader/doc/hbase20xsqlreader.md b/hbase20xsqlreader/doc/hbase20xsqlreader.md
index 9df020cc..43c42bf7 100644
--- a/hbase20xsqlreader/doc/hbase20xsqlreader.md
+++ b/hbase20xsqlreader/doc/hbase20xsqlreader.md
@@ -58,7 +58,9 @@ hbase20xsqlreader插件实现了从Phoenix(HBase SQL)读取数据，对应版本
* **queryServerAddress**
* 描述：hbase20xsqlreader需要通过Phoenix轻客户端去连接Phoenix QueryServer，因此这里需要填写对应QueryServer地址。
-
+ 增强版/Lindorm 用户若需透传user, password参数，可以在queryServerAddress后增加对应可选属性.
+ 格式参考：http://127.0.0.1:8765;user=root;password=root
+
* 必选：是
* 默认值：无
diff --git a/hbase20xsqlreader/pom.xml b/hbase20xsqlreader/pom.xml
index ec1c3419..818123f3 100644
--- a/hbase20xsqlreader/pom.xml
+++ b/hbase20xsqlreader/pom.xml
@@ -14,7 +14,7 @@
jar
- 5.1.0-HBase-2.0.0.2
+ 5.2.5-HBase-2.x
diff --git a/hbase20xsqlwriter/doc/hbase20xsqlwriter.md b/hbase20xsqlwriter/doc/hbase20xsqlwriter.md
index 2cc8cb41..e1f4e2f1 100644
--- a/hbase20xsqlwriter/doc/hbase20xsqlwriter.md
+++ b/hbase20xsqlwriter/doc/hbase20xsqlwriter.md
@@ -120,7 +120,9 @@ HBase20xsqlwriter实现了向hbase中的SQL表(phoenix)批量导入数据的功
* **queryServerAddress**
- * 描述：Phoenix QueryServer地址，为必填项，格式：http://${hostName}:${ip}，如http://172.16.34.58:8765
+ * 描述：Phoenix QueryServer地址，为必填项，格式：http://${hostName}:${ip}，如http://172.16.34.58:8765。
+ 增强版/Lindorm 用户若需透传user, password参数，可以在queryServerAddress后增加对应可选属性.
+ 格式参考：http://127.0.0.1:8765;user=root;password=root
* 必选：是
* 默认值：无
diff --git a/hbase20xsqlwriter/pom.xml b/hbase20xsqlwriter/pom.xml
index 690bc95e..5a2843e1 100644
--- a/hbase20xsqlwriter/pom.xml
+++ b/hbase20xsqlwriter/pom.xml
@@ -14,7 +14,7 @@
jar
- 5.1.0-HBase-2.0.0.2
+ 5.2.5-HBase-2.x
1.8
diff --git a/hbase20xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterTask.java b/hbase20xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterTask.java
index 43f710b7..481e07df 100644
--- a/hbase20xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterTask.java
+++ b/hbase20xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase20xsqlwriter/HBase20xSQLWriterTask.java
@@ -6,12 +6,12 @@ import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordReceiver;
import com.alibaba.datax.common.plugin.TaskPluginCollector;
import com.alibaba.datax.common.util.Configuration;
-import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
import java.sql.*;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -154,7 +154,7 @@ public class HBase20xSQLWriterTask {
* 从接收器中获取每条记录，写入Phoenix
*/
private void writeData(RecordReceiver lineReceiver) throws SQLException {
- List buffer = Lists.newArrayListWithExpectedSize(batchSize);
+ List buffer = new ArrayList(batchSize);
Record record = null;
while ((record = lineReceiver.getFromReader()) != null) {
// 校验列数量是否符合预期
diff --git a/hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/HdfsWriter.java b/hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/HdfsWriter.java
index 0119be2b..853613a2 100644
--- a/hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/HdfsWriter.java
+++ b/hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/HdfsWriter.java
@@ -81,10 +81,10 @@ public class HdfsWriter extends Writer {
//writeMode check
this.writeMode = this.writerSliceConfig.getNecessaryValue(Key.WRITE_MODE, HdfsWriterErrorCode.REQUIRED_VALUE);
writeMode = writeMode.toLowerCase().trim();
- Set supportedWriteModes = Sets.newHashSet("append", "nonconflict");
+ Set supportedWriteModes = Sets.newHashSet("append", "nonconflict", "truncate");
if (!supportedWriteModes.contains(writeMode)) {
throw DataXException.asDataXException(HdfsWriterErrorCode.ILLEGAL_VALUE,
- String.format("仅支持append, nonConflict两种模式, 不支持您配置的 writeMode 模式 : [%s]",
+ String.format("仅支持append, nonConflict, truncate三种模式, 不支持您配置的 writeMode 模式 : [%s]",
writeMode));
}
this.writerSliceConfig.set(Key.WRITE_MODE, writeMode);
@@ -179,6 +179,9 @@ public class HdfsWriter extends Writer {
LOG.error(String.format("冲突文件列表为: [%s]", StringUtils.join(allFiles, ",")));
throw DataXException.asDataXException(HdfsWriterErrorCode.ILLEGAL_VALUE,
String.format("由于您配置了writeMode nonConflict,但您配置的path: [%s] 目录不为空, 下面存在其他文件或文件夹.", path));
+ }else if ("truncate".equalsIgnoreCase(writeMode) && isExistFile) {
+ LOG.info(String.format("由于您配置了writeMode truncate, [%s] 下面的内容将被覆盖重写", path));
+ hdfsHelper.deleteFiles(existFilePaths);
}
}else{
throw DataXException.asDataXException(HdfsWriterErrorCode.ILLEGAL_VALUE,
diff --git a/images/DataX开源用户交流群.jpg b/images/DataX开源用户交流群.jpg
new file mode 100644
index 00000000..28155f38
Binary files /dev/null and b/images/DataX开源用户交流群.jpg differ
diff --git a/images/DataX开源用户交流群2.jpg b/images/DataX开源用户交流群2.jpg
new file mode 100644
index 00000000..129add96
Binary files /dev/null and b/images/DataX开源用户交流群2.jpg differ
diff --git a/images/DataX开源用户交流群3.jpg b/images/DataX开源用户交流群3.jpg
new file mode 100644
index 00000000..923061e8
Binary files /dev/null and b/images/DataX开源用户交流群3.jpg differ
diff --git a/images/DataX开源用户交流群4.jpg b/images/DataX开源用户交流群4.jpg
new file mode 100644
index 00000000..f0ce8db6
Binary files /dev/null and b/images/DataX开源用户交流群4.jpg differ
diff --git a/images/DataX开源用户交流群5.jpg b/images/DataX开源用户交流群5.jpg
new file mode 100644
index 00000000..1f3f7f95
Binary files /dev/null and b/images/DataX开源用户交流群5.jpg differ
diff --git a/introduction.md b/introduction.md
index b27607c7..d08ad98d 100644
--- a/introduction.md
+++ b/introduction.md
@@ -36,6 +36,7 @@ DataX本身作为离线数据同步框架，采用Framework + plugin架构构建
| ------------ | ---------- | :-------: | :-------: |:-------: |
| RDBMS 关系型数据库 | MySQL | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/mysqlreader/doc/mysqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/mysqlwriter/doc/mysqlwriter.md)|
| | Oracle | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/oraclereader/doc/oraclereader.md) 、[写](https://github.com/alibaba/DataX/blob/master/oraclewriter/doc/oraclewriter.md)|
+| | OceanBase | √ | √ |[读](https://open.oceanbase.com/docs/community/oceanbase-database/V3.1.0/use-datax-to-full-migration-data-to-oceanbase) 、[写](https://open.oceanbase.com/docs/community/oceanbase-database/V3.1.0/use-datax-to-full-migration-data-to-oceanbase)|
| | SQLServer | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/sqlserverreader/doc/sqlserverreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/sqlserverwriter/doc/sqlserverwriter.md)|
| | PostgreSQL | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/postgresqlreader/doc/postgresqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/postgresqlwriter/doc/postgresqlwriter.md)|
| | DRDS | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/drdsreader/doc/drdsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/drdswriter/doc/drdswriter.md)|
diff --git a/kingbaseesreader/doc/kingbaseesreader.md b/kingbaseesreader/doc/kingbaseesreader.md
new file mode 100644
index 00000000..ec5495a6
--- /dev/null
+++ b/kingbaseesreader/doc/kingbaseesreader.md
@@ -0,0 +1,241 @@
+
+# KingbaseesReader 插件文档
+
+
+___
+
+
+## 1 快速介绍
+
+KingbaseesReader插件实现了从KingbaseES读取数据。在底层实现上，KingbaseesReader通过JDBC连接远程KingbaseES数据库，并执行相应的sql语句将数据从KingbaseES库中SELECT出来。
+
+## 2 实现原理
+
+简而言之，KingbaseesReader通过JDBC连接器连接到远程的KingbaseES数据库，并根据用户配置的信息生成查询SELECT SQL语句并发送到远程KingbaseES数据库，并将该SQL执行返回结果使用DataX自定义的数据类型拼装为抽象的数据集，并传递给下游Writer处理。
+
+对于用户配置Table、Column、Where的信息，KingbaseesReader将其拼接为SQL语句发送到KingbaseES数据库；对于用户配置querySql信息，KingbaseesReader直接将其发送到KingbaseES数据库。
+
+
+## 3 功能说明
+
+### 3.1 配置样例
+
+* 配置一个从KingbaseES数据库同步抽取数据到本地的作业:
+
+```
+{
+ "job": {
+ "setting": {
+ "speed": {
+ //设置传输速度，单位为byte/s，DataX运行会尽可能达到该速度但是不超过它.
+ "byte": 1048576
+ },
+ //出错限制
+ "errorLimit": {
+ //出错的record条数上限，当大于该值即报错。
+ "record": 0,
+ //出错的record百分比上限 1.0表示100%，0.02表示2%
+ "percentage": 0.02
+ }
+ },
+ "content": [
+ {
+ "reader": {
+ "name": "kingbaseesreader",
+ "parameter": {
+ // 数据库连接用户名
+ "username": "xx",
+ // 数据库连接密码
+ "password": "xx",
+ "column": [
+ "id"，"name"
+ ],
+ //切分主键
+ "splitPk": "id",
+ "connection": [
+ {
+ "table": [
+ "table"
+ ],
+ "jdbcUrl": [
+ "jdbc:kingbase8://host:port/database"
+ ]
+ }
+ ]
+ }
+ },
+ "writer": {
+ //writer类型
+ "name": "streamwriter",
+ //是否打印内容
+ "parameter": {
+ "print":true,
+ }
+ }
+ }
+ ]
+ }
+}
+
+```
+
+* 配置一个自定义SQL的数据库同步任务到本地内容的作业：
+
+```
+{
+ "job": {
+ "setting": {
+ "speed": 1048576
+ },
+ "content": [
+ {
+ "reader": {
+ "name": "kingbaseesreader",
+ "parameter": {
+ "username": "xx",
+ "password": "xx",
+ "where": "",
+ "connection": [
+ {
+ "querySql": [
+ "select db_id,on_line_flag from db_info where db_id < 10;"
+ ],
+ "jdbcUrl": [
+ "jdbc:kingbase8://host:port/database", "jdbc:kingbase8://host:port/database"
+ ]
+ }
+ ]
+ }
+ },
+ "writer": {
+ "name": "streamwriter",
+ "parameter": {
+ "print": false,
+ "encoding": "UTF-8"
+ }
+ }
+ }
+ ]
+ }
+}
+```
+
+
+### 3.2 参数说明
+
+* **jdbcUrl**
+
+ * 描述：描述的是到对端数据库的JDBC连接信息，使用JSON的数组描述，并支持一个库填写多个连接地址。之所以使用JSON数组描述连接信息，是因为阿里集团内部支持多个IP探测，如果配置了多个，KingbaseesReader可以依次探测ip的可连接性，直到选择一个合法的IP。如果全部连接失败，KingbaseesReader报错。 注意，jdbcUrl必须包含在connection配置单元中。对于阿里集团外部使用情况，JSON数组填写一个JDBC连接即可。
+
+ jdbcUrl按照KingbaseES官方规范，并可以填写连接附件控制信息。具体请参看[KingbaseES官方文档](https://help.kingbase.com.cn/doc-view-5683.html)。
+
+ * 必选：是
+
+ * 默认值：无
+
+* **username**
+
+ * 描述：数据源的用户名
+
+ * 必选：是
+
+ * 默认值：无
+
+* **password**
+
+ * 描述：数据源指定用户名的密码
+
+ * 必选：是
+
+ * 默认值：无
+
+* **table**
+
+ * 描述：所选取的需要同步的表。使用JSON的数组描述，因此支持多张表同时抽取。当配置为多张表时，用户自己需保证多张表是同一schema结构，KingbaseesReader不予检查表是否同一逻辑表。注意，table必须包含在connection配置单元中。
+
+ * 必选：是
+
+ * 默认值：无
+
+* **column**
+
+ * 描述：所配置的表中需要同步的列名集合，使用JSON的数组描述字段信息。用户使用\*代表默认使用所有列配置，例如['\*']。
+
+ 支持列裁剪，即列可以挑选部分列进行导出。
+
+ 支持列换序，即列可以不按照表schema信息进行导出。
+
+ 支持常量配置，用户需要按照KingbaseES语法格式:
+ ["id", "'hello'::varchar", "true", "2.5::real", "power(2,3)"]
+ id为普通列名，'hello'::varchar为字符串常量，true为布尔值，2.5为浮点数, power(2,3)为函数。
+
+ **column必须用户显示指定同步的列集合，不允许为空！**
+
+ * 必选：是
+
+ * 默认值：无
+
+* **splitPk**
+
+ * 描述：KingbaseesReader进行数据抽取时，如果指定splitPk，表示用户希望使用splitPk代表的字段进行数据分片，DataX因此会启动并发任务进行数据同步，这样可以大大提供数据同步的效能。
+
+ 推荐splitPk用户使用表主键，因为表主键通常情况下比较均匀，因此切分出来的分片也不容易出现数据热点。
+
+ 目前splitPk仅支持整形数据切分，`不支持浮点、字符串型、日期等其他类型`。如果用户指定其他非支持类型，KingbaseesReader将报错！
+
+ splitPk设置为空，底层将视作用户不允许对单表进行切分，因此使用单通道进行抽取。
+
+ * 必选：否
+
+ * 默认值：空
+
+* **where**
+
+ * 描述：筛选条件，KingbaseesReader根据指定的column、table、where条件拼接SQL，并根据这个SQL进行数据抽取。在实际业务场景中，往往会选择当天的数据进行同步，可以将where条件指定为gmt_create > $bizdate 。注意：不可以将where条件指定为limit 10，limit不是SQL的合法where子句。
+
+ where条件可以有效地进行业务增量同步。 where条件不配置或者为空，视作全表同步数据。
+
+ * 必选：否
+
+ * 默认值：无
+
+* **querySql**
+
+ * 描述：在有些业务场景下，where这一配置项不足以描述所筛选的条件，用户可以通过该配置型来自定义筛选SQL。当用户配置了这一项之后，DataX系统就会忽略table，column这些配置型，直接使用这个配置项的内容对数据进行筛选，例如需要进行多表join后同步数据，使用select a,b from table_a join table_b on table_a.id = table_b.id
+
+ `当用户配置querySql时，KingbaseesReader直接忽略table、column、where条件的配置`。
+
+ * 必选：否
+
+ * 默认值：无
+
+* **fetchSize**
+
+ * 描述：该配置项定义了插件和数据库服务器端每次批量数据获取条数，该值决定了DataX和服务器端的网络交互次数，能够较大的提升数据抽取性能。
+
+ `注意，该值过大(>2048)可能造成DataX进程OOM。`。
+
+ * 必选：否
+
+ * 默认值：1024
+
+
+### 3.3 类型转换
+
+目前KingbaseesReader支持大部分KingbaseES类型，但也存在部分个别类型没有支持的情况，请注意检查你的类型。
+
+下面列出KingbaseesReader针对KingbaseES类型转换列表:
+
+
+| DataX 内部类型| KingbaseES 数据类型 |
+| -------- | ----- |
+| Long |bigint, bigserial, integer, smallint, serial |
+| Double |double precision, money, numeric, real |
+| String |varchar, char, text, bit, inet|
+| Date |date, time, timestamp |
+| Boolean |bool|
+| Bytes |bytea|
+
+请注意:
+
+* `除上述罗列字段类型外，其他类型均不支持; money,inet,bit需用户使用a_inet::varchar类似的语法转换`。
\ No newline at end of file
diff --git a/kingbaseesreader/pom.xml b/kingbaseesreader/pom.xml
new file mode 100644
index 00000000..6e844c10
--- /dev/null
+++ b/kingbaseesreader/pom.xml
@@ -0,0 +1,88 @@
+
+
+
+ datax-all
+ com.alibaba.datax
+ 0.0.1-SNAPSHOT
+
+ 4.0.0
+
+ kingbaseesreader
+ kingbaseesreader
+ jar
+
+
+
+ com.alibaba.datax
+ datax-common
+ ${datax-project-version}
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+ ch.qos.logback
+ logback-classic
+
+
+
+ com.alibaba.datax
+ plugin-rdbms-util
+ ${datax-project-version}
+
+
+
+ com.kingbase8
+ kingbase8
+ 8.2.0
+ system
+ ${basedir}/src/main/libs/kingbase8-8.2.0.jar
+
+
+
+
+
+
+
+
+ maven-compiler-plugin
+
+ ${jdk-version}
+ ${jdk-version}
+ ${project-sourceEncoding}
+
+
+
+
+ maven-assembly-plugin
+
+
+ src/main/assembly/package.xml
+
+ datax
+
+
+
+ dwzip
+ package
+
+ single
+
+
+
+
+
+
+
+
diff --git a/kingbaseesreader/src/main/assembly/package.xml b/kingbaseesreader/src/main/assembly/package.xml
new file mode 100644
index 00000000..e369c5f0
--- /dev/null
+++ b/kingbaseesreader/src/main/assembly/package.xml
@@ -0,0 +1,42 @@
+
+
+
+ dir
+
+ false
+
+
+ src/main/resources
+
+ plugin.json
+ plugin_job_template.json
+
+ plugin/reader/kingbaseesreader
+
+
+ target/
+
+ kingbaseesreader-0.0.1-SNAPSHOT.jar
+
+ plugin/reader/kingbaseesreader
+
+
+ src/main/libs
+
+ *.*
+
+ plugin/reader/kingbaseesreader/libs
+
+
+
+
+
+ false
+ plugin/reader/kingbaseesreader/libs
+ runtime
+
+
+
diff --git a/kingbaseesreader/src/main/java/com/alibaba/datax/plugin/reader/kingbaseesreader/Constant.java b/kingbaseesreader/src/main/java/com/alibaba/datax/plugin/reader/kingbaseesreader/Constant.java
new file mode 100644
index 00000000..bed4c6e6
--- /dev/null
+++ b/kingbaseesreader/src/main/java/com/alibaba/datax/plugin/reader/kingbaseesreader/Constant.java
@@ -0,0 +1,7 @@
+package com.alibaba.datax.plugin.reader.kingbaseesreader;
+
+public class Constant {
+
+ public static final int DEFAULT_FETCH_SIZE = 1000;
+
+}
diff --git a/kingbaseesreader/src/main/java/com/alibaba/datax/plugin/reader/kingbaseesreader/KingbaseesReader.java b/kingbaseesreader/src/main/java/com/alibaba/datax/plugin/reader/kingbaseesreader/KingbaseesReader.java
new file mode 100644
index 00000000..9246655f
--- /dev/null
+++ b/kingbaseesreader/src/main/java/com/alibaba/datax/plugin/reader/kingbaseesreader/KingbaseesReader.java
@@ -0,0 +1,86 @@
+package com.alibaba.datax.plugin.reader.kingbaseesreader;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordSender;
+import com.alibaba.datax.common.spi.Reader;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.rdbms.reader.CommonRdbmsReader;
+import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
+import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
+
+import java.util.List;
+
+public class KingbaseesReader extends Reader {
+
+ private static final DataBaseType DATABASE_TYPE = DataBaseType.KingbaseES;
+
+ public static class Job extends Reader.Job {
+
+ private Configuration originalConfig;
+ private CommonRdbmsReader.Job commonRdbmsReaderMaster;
+
+ @Override
+ public void init() {
+ this.originalConfig = super.getPluginJobConf();
+ int fetchSize = this.originalConfig.getInt(com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE,
+ Constant.DEFAULT_FETCH_SIZE);
+ if (fetchSize < 1) {
+ throw DataXException.asDataXException(DBUtilErrorCode.REQUIRED_VALUE,
+ String.format("您配置的fetchSize有误，根据DataX的设计，fetchSize : [%d] 设置值不能小于 1.", fetchSize));
+ }
+ this.originalConfig.set(com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE, fetchSize);
+
+ this.commonRdbmsReaderMaster = new CommonRdbmsReader.Job(DATABASE_TYPE);
+ this.commonRdbmsReaderMaster.init(this.originalConfig);
+ }
+
+ @Override
+ public List split(int adviceNumber) {
+ return this.commonRdbmsReaderMaster.split(this.originalConfig, adviceNumber);
+ }
+
+ @Override
+ public void post() {
+ this.commonRdbmsReaderMaster.post(this.originalConfig);
+ }
+
+ @Override
+ public void destroy() {
+ this.commonRdbmsReaderMaster.destroy(this.originalConfig);
+ }
+
+ }
+
+ public static class Task extends Reader.Task {
+
+ private Configuration readerSliceConfig;
+ private CommonRdbmsReader.Task commonRdbmsReaderSlave;
+
+ @Override
+ public void init() {
+ this.readerSliceConfig = super.getPluginJobConf();
+ this.commonRdbmsReaderSlave = new CommonRdbmsReader.Task(DATABASE_TYPE, super.getTaskGroupId(), super.getTaskId());
+ this.commonRdbmsReaderSlave.init(this.readerSliceConfig);
+ }
+
+ @Override
+ public void startRead(RecordSender recordSender) {
+ int fetchSize = this.readerSliceConfig.getInt(com.alibaba.datax.plugin.rdbms.reader.Constant.FETCH_SIZE);
+
+ this.commonRdbmsReaderSlave.startRead(this.readerSliceConfig, recordSender,
+ super.getTaskPluginCollector(), fetchSize);
+ }
+
+ @Override
+ public void post() {
+ this.commonRdbmsReaderSlave.post(this.readerSliceConfig);
+ }
+
+ @Override
+ public void destroy() {
+ this.commonRdbmsReaderSlave.destroy(this.readerSliceConfig);
+ }
+
+ }
+
+}
diff --git a/kingbaseesreader/src/main/libs/kingbase8-8.2.0.jar b/kingbaseesreader/src/main/libs/kingbase8-8.2.0.jar
new file mode 100644
index 00000000..0b5ac964
Binary files /dev/null and b/kingbaseesreader/src/main/libs/kingbase8-8.2.0.jar differ
diff --git a/kingbaseesreader/src/main/resources/plugin.json b/kingbaseesreader/src/main/resources/plugin.json
new file mode 100644
index 00000000..9bc0684b
--- /dev/null
+++ b/kingbaseesreader/src/main/resources/plugin.json
@@ -0,0 +1,6 @@
+{
+ "name": "kingbaseesreader",
+ "class": "com.alibaba.datax.plugin.reader.kingbaseesreader.KingbaseesReader",
+ "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
+ "developer": "alibaba"
+}
\ No newline at end of file
diff --git a/kingbaseesreader/src/main/resources/plugin_job_template.json b/kingbaseesreader/src/main/resources/plugin_job_template.json
new file mode 100644
index 00000000..49c07098
--- /dev/null
+++ b/kingbaseesreader/src/main/resources/plugin_job_template.json
@@ -0,0 +1,13 @@
+{
+ "name": "kingbaseesreader",
+ "parameter": {
+ "username": "",
+ "password": "",
+ "connection": [
+ {
+ "table": [],
+ "jdbcUrl": []
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/kingbaseeswriter/doc/kingbaseeswriter.md b/kingbaseeswriter/doc/kingbaseeswriter.md
new file mode 100644
index 00000000..96e1a3ac
--- /dev/null
+++ b/kingbaseeswriter/doc/kingbaseeswriter.md
@@ -0,0 +1,208 @@
+# DataX KingbaseesWriter
+
+
+---
+
+
+## 1 快速介绍
+
+KingbaseesWriter插件实现了写入数据到 KingbaseES主库目的表的功能。在底层实现上，KingbaseesWriter通过JDBC连接远程 KingbaseES 数据库，并执行相应的 insert into ... sql 语句将数据写入 KingbaseES，内部会分批次提交入库。
+
+KingbaseesWriter面向ETL开发工程师，他们使用KingbaseesWriter从数仓导入数据到KingbaseES。同时 KingbaseesWriter亦可以作为数据迁移工具为DBA等用户提供服务。
+
+
+## 2 实现原理
+
+KingbaseesWriter通过 DataX 框架获取 Reader 生成的协议数据，根据你配置生成相应的SQL插入语句
+
+
+* `insert into...`(当主键/唯一性索引冲突时会写不进去冲突的行)
+
+
+
+ 注意：
+ 1. 目的表所在数据库必须是主库才能写入数据；整个任务至少需具备 insert into...的权限，是否需要其他权限，取决于你任务配置中在 preSql 和 postSql 中指定的语句。
+ 2. KingbaseesWriter和MysqlWriter不同，不支持配置writeMode参数。
+
+
+## 3 功能说明
+
+### 3.1 配置样例
+
+* 这里使用一份从内存产生到 KingbaseesWriter导入的数据。
+
+```json
+{
+ "job": {
+ "setting": {
+ "speed": {
+ "channel": 1
+ }
+ },
+ "content": [
+ {
+ "reader": {
+ "name": "streamreader",
+ "parameter": {
+ "column" : [
+ {
+ "value": "DataX",
+ "type": "string"
+ },
+ {
+ "value": 19880808,
+ "type": "long"
+ },
+ {
+ "value": "1988-08-08 08:08:08",
+ "type": "date"
+ },
+ {
+ "value": true,
+ "type": "bool"
+ },
+ {
+ "value": "test",
+ "type": "bytes"
+ }
+ ],
+ "sliceRecordCount": 1000
+ }
+ },
+ "writer": {
+ "name": "kingbaseeswriter",
+ "parameter": {
+ "username": "xx",
+ "password": "xx",
+ "column": [
+ "id",
+ "name"
+ ],
+ "preSql": [
+ "delete from test"
+ ],
+ "connection": [
+ {
+ "jdbcUrl": "jdbc:kingbase8://127.0.0.1:3002/datax",
+ "table": [
+ "test"
+ ]
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+}
+
+```
+
+
+### 3.2 参数说明
+
+* **jdbcUrl**
+
+ * 描述：目的数据库的 JDBC 连接信息 ,jdbcUrl必须包含在connection配置单元中。
+
+ 注意：1、在一个数据库上只能配置一个值。
+ 2、jdbcUrl按照KingbaseES官方规范，并可以填写连接附加参数信息。具体请参看KingbaseES官方文档或者咨询对应 DBA。
+
+
+ * 必选：是
+
+ * 默认值：无
+
+* **username**
+
+ * 描述：目的数据库的用户名
+
+ * 必选：是
+
+ * 默认值：无
+
+* **password**
+
+ * 描述：目的数据库的密码
+
+ * 必选：是
+
+ * 默认值：无
+
+* **table**
+
+ * 描述：目的表的表名称。支持写入一个或者多个表。当配置为多张表时，必须确保所有表结构保持一致。
+
+ 注意：table 和 jdbcUrl 必须包含在 connection 配置单元中
+
+ * 必选：是
+
+ * 默认值：无
+
+* **column**
+
+ * 描述：目的表需要写入数据的字段,字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。如果要依次写入全部列，使用\*表示, 例如: "column": ["\*"]
+
+ 注意：1、我们强烈不推荐你这样配置，因为当你目的表字段个数、类型等有改动时，你的任务可能运行不正确或者失败
+ 2、此处 column 不能配置任何常量值
+
+ * 必选：是
+
+ * 默认值：否
+
+* **preSql**
+
+ * 描述：写入数据到目的表前，会先执行这里的标准语句。如果 Sql 中有你需要操作到的表名称，请使用 `@table` 表示，这样在实际执行 Sql 语句时，会对变量按照实际表名称进行替换。比如你的任务是要写入到目的端的100个同构分表(表名称为:datax_00,datax01, ... datax_98,datax_99)，并且你希望导入数据前，先对表中数据进行删除操作，那么你可以这样配置：`"preSql":["delete from @table"]`，效果是：在执行到每个表写入数据前，会先执行对应的 delete from 对应表名称
+
+ * 必选：否
+
+ * 默认值：无
+
+* **postSql**
+
+ * 描述：写入数据到目的表后，会执行这里的标准语句。（原理同 preSql ）
+
+ * 必选：否
+
+ * 默认值：无
+
+* **batchSize**
+
+ * 描述：一次性批量提交的记录数大小，该值可以极大减少DataX与KingbaseES的网络交互次数，并提升整体吞吐量。但是该值设置过大可能会造成DataX运行进程OOM情况。
+
+ * 必选：否
+
+ * 默认值：1024
+
+### 3.3 类型转换
+
+目前 KingbaseesWriter支持大部分 KingbaseES类型，但也存在部分没有支持的情况，请注意检查你的类型。
+
+下面列出 KingbaseesWriter针对 KingbaseES类型转换列表:
+
+| DataX 内部类型| KingbaseES 数据类型 |
+| -------- | ----- |
+| Long |bigint, bigserial, integer, smallint, serial |
+| Double |double precision, money, numeric, real |
+| String |varchar, char, text, bit|
+| Date |date, time, timestamp |
+| Boolean |bool|
+| Bytes |bytea|
+
+
+## FAQ
+
+***
+
+**Q: KingbaseesWriter 执行 postSql 语句报错，那么数据导入到目标数据库了吗?**
+
+A: DataX 导入过程存在三块逻辑，pre 操作、导入操作、post 操作，其中任意一环报错，DataX 作业报错。由于 DataX 不能保证在同一个事务完成上述几个操作，因此有可能数据已经落入到目标端。
+
+***
+
+**Q: 按照上述说法，那么有部分脏数据导入数据库，如果影响到线上数据库怎么办?**
+
+A: 目前有两种解法，第一种配置 pre 语句，该 sql 可以清理当天导入数据， DataX 每次导入时候可以把上次清理干净并导入完整数据。
+第二种，向临时表导入数据，完成后再 rename 到线上表。
+
+***
diff --git a/kingbaseeswriter/pom.xml b/kingbaseeswriter/pom.xml
new file mode 100644
index 00000000..284c8c5e
--- /dev/null
+++ b/kingbaseeswriter/pom.xml
@@ -0,0 +1,84 @@
+
+ 4.0.0
+
+ com.alibaba.datax
+ datax-all
+ 0.0.1-SNAPSHOT
+
+ kingbaseeswriter
+ kingbaseeswriter
+ jar
+ writer data into kingbasees database
+
+
+
+ com.alibaba.datax
+ datax-common
+ ${datax-project-version}
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+ ch.qos.logback
+ logback-classic
+
+
+
+ com.alibaba.datax
+ plugin-rdbms-util
+ ${datax-project-version}
+
+
+
+ com.kingbase8
+ kingbase8
+ 8.2.0
+ system
+ ${basedir}/src/main/libs/kingbase8-8.2.0.jar
+
+
+
+
+
+
+
+ maven-compiler-plugin
+
+ ${jdk-version}
+ ${jdk-version}
+ ${project-sourceEncoding}
+
+
+
+
+ maven-assembly-plugin
+
+
+ src/main/assembly/package.xml
+
+ datax
+
+
+
+ dwzip
+ package
+
+ single
+
+
+
+
+
+
+
diff --git a/kingbaseeswriter/src/main/assembly/package.xml b/kingbaseeswriter/src/main/assembly/package.xml
new file mode 100644
index 00000000..aa78a6ec
--- /dev/null
+++ b/kingbaseeswriter/src/main/assembly/package.xml
@@ -0,0 +1,42 @@
+
+
+
+ dir
+
+ false
+
+
+ src/main/resources
+
+ plugin.json
+ plugin_job_template.json
+
+ plugin/writer/kingbaseeswriter
+
+
+ target/
+
+ kingbaseeswriter-0.0.1-SNAPSHOT.jar
+
+ plugin/writer/kingbaseeswriter
+
+
+ src/main/libs
+
+ *.*
+
+ plugin/writer/kingbaseeswriter/libs
+
+
+
+
+
+ false
+ plugin/writer/kingbaseeswriter/libs
+ runtime
+
+
+
diff --git a/kingbaseeswriter/src/main/java/com/alibaba/datax/plugin/writer/kingbaseeswriter/KingbaseesWriter.java b/kingbaseeswriter/src/main/java/com/alibaba/datax/plugin/writer/kingbaseeswriter/KingbaseesWriter.java
new file mode 100644
index 00000000..dec5ff95
--- /dev/null
+++ b/kingbaseeswriter/src/main/java/com/alibaba/datax/plugin/writer/kingbaseeswriter/KingbaseesWriter.java
@@ -0,0 +1,100 @@
+package com.alibaba.datax.plugin.writer.kingbaseeswriter;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+import com.alibaba.datax.common.spi.Writer;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
+import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
+import com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter;
+import com.alibaba.datax.plugin.rdbms.writer.Key;
+
+import java.util.List;
+
+public class KingbaseesWriter extends Writer {
+ private static final DataBaseType DATABASE_TYPE = DataBaseType.KingbaseES;
+
+ public static class Job extends Writer.Job {
+ private Configuration originalConfig = null;
+ private CommonRdbmsWriter.Job commonRdbmsWriterMaster;
+
+ @Override
+ public void init() {
+ this.originalConfig = super.getPluginJobConf();
+
+ // warn：not like mysql, KingbaseES only support insert mode, don't use
+ String writeMode = this.originalConfig.getString(Key.WRITE_MODE);
+ if (null != writeMode) {
+ throw DataXException.asDataXException(DBUtilErrorCode.CONF_ERROR,
+ String.format("写入模式(writeMode)配置有误. 因为KingbaseES不支持配置参数项 writeMode: %s, KingbaseES仅使用insert sql 插入数据. 请检查您的配置并作出修改.", writeMode));
+ }
+
+ this.commonRdbmsWriterMaster = new CommonRdbmsWriter.Job(DATABASE_TYPE);
+ this.commonRdbmsWriterMaster.init(this.originalConfig);
+ }
+
+ @Override
+ public void prepare() {
+ this.commonRdbmsWriterMaster.prepare(this.originalConfig);
+ }
+
+ @Override
+ public List split(int mandatoryNumber) {
+ return this.commonRdbmsWriterMaster.split(this.originalConfig, mandatoryNumber);
+ }
+
+ @Override
+ public void post() {
+ this.commonRdbmsWriterMaster.post(this.originalConfig);
+ }
+
+ @Override
+ public void destroy() {
+ this.commonRdbmsWriterMaster.destroy(this.originalConfig);
+ }
+
+ }
+
+ public static class Task extends Writer.Task {
+ private Configuration writerSliceConfig;
+ private CommonRdbmsWriter.Task commonRdbmsWriterSlave;
+
+ @Override
+ public void init() {
+ this.writerSliceConfig = super.getPluginJobConf();
+ this.commonRdbmsWriterSlave = new CommonRdbmsWriter.Task(DATABASE_TYPE){
+ @Override
+ public String calcValueHolder(String columnType){
+ if("serial".equalsIgnoreCase(columnType)){
+ return "?::int";
+ }else if("bit".equalsIgnoreCase(columnType)){
+ return "?::bit varying";
+ }
+ return "?::" + columnType;
+ }
+ };
+ this.commonRdbmsWriterSlave.init(this.writerSliceConfig);
+ }
+
+ @Override
+ public void prepare() {
+ this.commonRdbmsWriterSlave.prepare(this.writerSliceConfig);
+ }
+
+ public void startWrite(RecordReceiver recordReceiver) {
+ this.commonRdbmsWriterSlave.startWrite(recordReceiver, this.writerSliceConfig, super.getTaskPluginCollector());
+ }
+
+ @Override
+ public void post() {
+ this.commonRdbmsWriterSlave.post(this.writerSliceConfig);
+ }
+
+ @Override
+ public void destroy() {
+ this.commonRdbmsWriterSlave.destroy(this.writerSliceConfig);
+ }
+
+ }
+
+}
diff --git a/kingbaseeswriter/src/main/libs/kingbase8-8.2.0.jar b/kingbaseeswriter/src/main/libs/kingbase8-8.2.0.jar
new file mode 100644
index 00000000..0b5ac964
Binary files /dev/null and b/kingbaseeswriter/src/main/libs/kingbase8-8.2.0.jar differ
diff --git a/kingbaseeswriter/src/main/resources/plugin.json b/kingbaseeswriter/src/main/resources/plugin.json
new file mode 100644
index 00000000..83517760
--- /dev/null
+++ b/kingbaseeswriter/src/main/resources/plugin.json
@@ -0,0 +1,6 @@
+{
+ "name": "kingbaseeswriter",
+ "class": "com.alibaba.datax.plugin.writer.kingbaseeswriter.KingbaseesWriter",
+ "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.",
+ "developer": "alibaba"
+}
\ No newline at end of file
diff --git a/kingbaseeswriter/src/main/resources/plugin_job_template.json b/kingbaseeswriter/src/main/resources/plugin_job_template.json
new file mode 100644
index 00000000..94b66168
--- /dev/null
+++ b/kingbaseeswriter/src/main/resources/plugin_job_template.json
@@ -0,0 +1,17 @@
+{
+ "name": "kingbaseeswriter",
+ "parameter": {
+ "username": "",
+ "password": "",
+ "column": [],
+ "preSql": [],
+ "connection": [
+ {
+ "jdbcUrl": "",
+ "table": []
+ }
+ ],
+ "preSql": [],
+ "postSql": []
+ }
+}
\ No newline at end of file
diff --git a/kuduwriter/README.md b/kuduwriter/README.md
new file mode 100644
index 00000000..f53de1b5
--- /dev/null
+++ b/kuduwriter/README.md
@@ -0,0 +1,6 @@
+# datax-kudu-plugin
+datax kudu的writer插件
+
+
+
+仅在kudu11进行过测试
diff --git a/kuduwriter/doc/image-20200901193148188.png b/kuduwriter/doc/image-20200901193148188.png
new file mode 100644
index 00000000..7e7b8a1f
Binary files /dev/null and b/kuduwriter/doc/image-20200901193148188.png differ
diff --git a/kuduwriter/doc/kuduwirter.md b/kuduwriter/doc/kuduwirter.md
new file mode 100644
index 00000000..1a952449
--- /dev/null
+++ b/kuduwriter/doc/kuduwirter.md
@@ -0,0 +1,143 @@
+# datax-kudu-plugins
+datax kudu的writer插件
+
+
+
+eg:
+
+```json
+{
+ "name": "kuduwriter",
+ "parameter": {
+ "kuduConfig": {
+ "kudu.master_addresses": "***",
+ "timeout": 60000,
+ "sessionTimeout": 60000
+
+ },
+ "table": "",
+ "replicaCount": 3,
+ "truncate": false,
+ "writeMode": "upsert",
+ "partition": {
+ "range": {
+ "column1": [
+ {
+ "lower": "2020-08-25",
+ "upper": "2020-08-26"
+ },
+ {
+ "lower": "2020-08-26",
+ "upper": "2020-08-27"
+ },
+ {
+ "lower": "2020-08-27",
+ "upper": "2020-08-28"
+ }
+ ]
+ },
+ "hash": {
+ "column": [
+ "column1"
+ ],
+ "number": 3
+ }
+ },
+ "column": [
+ {
+ "index": 0,
+ "name": "c1",
+ "type": "string",
+ "primaryKey": true
+ },
+ {
+ "index": 1,
+ "name": "c2",
+ "type": "string",
+ "compress": "DEFAULT_COMPRESSION",
+ "encoding": "AUTO_ENCODING",
+ "comment": "注解xxxx"
+ }
+ ],
+ "batchSize": 1024,
+ "bufferSize": 2048,
+ "skipFail": false,
+ "encoding": "UTF-8"
+ }
+}
+```
+
+必须参数：
+
+```json
+ "writer": {
+ "name": "kuduwriter",
+ "parameter": {
+ "kuduConfig": {
+ "kudu.master_addresses": "***"
+ },
+ "table": "***",
+ "column": [
+ {
+ "name": "c1",
+ "type": "string",
+ "primaryKey": true
+ },
+ {
+ "name": "c2",
+ "type": "string",
+ },
+ {
+ "name": "c3",
+ "type": "string"
+ },
+ {
+ "name": "c4",
+ "type": "string"
+ }
+ ]
+ }
+ }
+```
+
+主键列请写到最前面
+
+
+
+
+
+##### 配置列表
+
+| name | default | description | 是否必须 |
+| -------------- | ------------------- | ------------------------------------------------------------ | -------- |
+| kuduConfig | | kudu配置 （kudu.master_addresses等） | 是 |
+| table | | 导入目标表名 | 是 |
+| partition | | 分区 | 否 |
+| column | | 列 | 是 |
+| name | | 列名 | 是 |
+| type | string | 列的类型，现支持INT, FLOAT, STRING, BIGINT, DOUBLE, BOOLEAN, LONG。 | 否 |
+| index | 升序排列 | 列索引位置(要么全部列都写，要么都不写)，如reader中取到的某一字段在第二位置（eg： name， id， age）但kudu目标表结构不同（eg：id，name， age），此时就需要将index赋值为（1，0，2），默认顺序（0，1，2） | 否 |
+| primaryKey | false | 是否为主键（请将所有的主键列写在前面）,不表明主键将不会检查过滤脏数据 | 否 |
+| compress | DEFAULT_COMPRESSION | 压缩格式 | 否 |
+| encoding | AUTO_ENCODING | 编码 | 否 |
+| replicaCount | 3 | 保留副本个数 | 否 |
+| hash | | hash分区 | 否 |
+| number | 3 | hash分区个数 | 否 |
+| range | | range分区 | 否 |
+| lower | | range分区下限 (eg: sql建表：partition value='haha' 对应：“lower”：“haha”，“upper”：“haha\000”) | 否 |
+| upper | | range分区上限(eg: sql建表：partition "10" <= VALUES < "20" 对应：“lower”：“10”，“upper”：“20”) | 否 |
+| truncate | false | 是否清空表，本质上是删表重建 | 否 |
+| writeMode | upsert | upsert，insert，update | 否 |
+| batchSize | 512 | 每xx行数据flush一次结果（最好不要超过1024） | 否 |
+| bufferSize | 3072 | 缓冲区大小 | 否 |
+| skipFail | false | 是否跳过插入不成功的数据 | 否 |
+| timeout | 60000 | client超时时间,如创建表，删除表操作的超时时间。单位：ms | 否 |
+| sessionTimeout | 60000 | session超时时间 单位：ms | 否 |
+
+
+
+
+
+
+
+
diff --git a/kuduwriter/pom.xml b/kuduwriter/pom.xml
new file mode 100644
index 00000000..5d78be4c
--- /dev/null
+++ b/kuduwriter/pom.xml
@@ -0,0 +1,82 @@
+
+
+
+ datax-all
+ com.alibaba.datax
+ 0.0.1-SNAPSHOT
+
+ 4.0.0
+
+ kuduwriter
+
+
+ com.alibaba.datax
+ datax-common
+ ${datax-project-version}
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
+
+
+ org.apache.kudu
+ kudu-client
+ 1.11.1
+
+
+ junit
+ junit
+ 4.13.1
+ test
+
+
+ com.alibaba.datax
+ datax-core
+ ${datax-project-version}
+
+
+ com.alibaba.datax
+ datax-service-face
+
+
+ test
+
+
+
+
+
+
+
+ maven-compiler-plugin
+
+ ${jdk-version}
+ ${jdk-version}
+ ${project-sourceEncoding}
+
+
+
+
+ maven-assembly-plugin
+
+
+ src/main/assembly/package.xml
+
+ datax
+
+
+
+ dwzip
+ package
+
+ single
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/kuduwriter/src/main/assembly/package.xml b/kuduwriter/src/main/assembly/package.xml
new file mode 100644
index 00000000..c9497b92
--- /dev/null
+++ b/kuduwriter/src/main/assembly/package.xml
@@ -0,0 +1,35 @@
+
+
+
+ dir
+
+ false
+
+
+ src/main/resources
+
+ plugin.json
+ plugin_job_template.json
+
+ plugin/writer/kuduwriter
+
+
+ target/
+
+ kuduwriter-0.0.1-SNAPSHOT.jar
+
+ plugin/writer/kuduwriter
+
+
+
+
+
+ false
+ plugin/writer/kuduwriter/libs
+ runtime
+
+
+
\ No newline at end of file
diff --git a/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/ColumnType.java b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/ColumnType.java
new file mode 100644
index 00000000..ebd6ea79
--- /dev/null
+++ b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/ColumnType.java
@@ -0,0 +1,37 @@
+package com.q1.datax.plugin.writer.kudu11xwriter;
+
+import com.alibaba.datax.common.exception.DataXException;
+
+import java.util.Arrays;
+
+/**
+ * @author daizihao
+ * @create 2020-08-31 19:12
+ **/
+public enum ColumnType {
+ INT("int"),
+ FLOAT("float"),
+ STRING("string"),
+ BIGINT("bigint"),
+ DOUBLE("double"),
+ BOOLEAN("boolean"),
+ LONG("long");
+ private String mode;
+ ColumnType(String mode) {
+ this.mode = mode.toLowerCase();
+ }
+
+ public String getMode() {
+ return mode;
+ }
+
+ public static ColumnType getByTypeName(String modeName) {
+ for (ColumnType modeType : values()) {
+ if (modeType.mode.equalsIgnoreCase(modeName)) {
+ return modeType;
+ }
+ }
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.ILLEGAL_VALUE,
+ String.format("Kuduwriter does not support the type:%s, currently supported types are:%s", modeName, Arrays.asList(values())));
+ }
+}
diff --git a/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Constant.java b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Constant.java
new file mode 100644
index 00000000..2710e350
--- /dev/null
+++ b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Constant.java
@@ -0,0 +1,21 @@
+package com.q1.datax.plugin.writer.kudu11xwriter;
+
+/**
+ * @author daizihao
+ * @create 2020-08-31 14:42
+ **/
+public class Constant {
+ public static final String DEFAULT_ENCODING = "UTF-8";
+// public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss";
+
+ public static final String COMPRESSION = "DEFAULT_COMPRESSION";
+ public static final String ENCODING = "AUTO_ENCODING";
+ public static final Long ADMIN_TIMEOUTMS = 60000L;
+ public static final Long SESSION_TIMEOUTMS = 60000L;
+
+
+ public static final String INSERT_MODE = "upsert";
+ public static final long DEFAULT_WRITE_BATCH_SIZE = 512L;
+ public static final long DEFAULT_MUTATION_BUFFER_SPACE = 3072L;
+
+}
diff --git a/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/InsertModeType.java b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/InsertModeType.java
new file mode 100644
index 00000000..754ca4fc
--- /dev/null
+++ b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/InsertModeType.java
@@ -0,0 +1,34 @@
+package com.q1.datax.plugin.writer.kudu11xwriter;
+
+import com.alibaba.datax.common.exception.DataXException;
+
+import java.util.Arrays;
+
+/**
+ * @author daizihao
+ * @create 2020-08-31 14:47
+ **/
+public enum InsertModeType {
+ Insert("insert"),
+ Upsert("upsert"),
+ Update("update");
+ private String mode;
+
+ InsertModeType(String mode) {
+ this.mode = mode.toLowerCase();
+ }
+
+ public String getMode() {
+ return mode;
+ }
+
+ public static InsertModeType getByTypeName(String modeName) {
+ for (InsertModeType modeType : values()) {
+ if (modeType.mode.equalsIgnoreCase(modeName)) {
+ return modeType;
+ }
+ }
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.ILLEGAL_VALUE,
+ String.format("Kuduwriter does not support the mode :[%s], currently supported mode types are :%s", modeName, Arrays.asList(values())));
+ }
+}
diff --git a/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Key.java b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Key.java
new file mode 100644
index 00000000..7e5755aa
--- /dev/null
+++ b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Key.java
@@ -0,0 +1,45 @@
+package com.q1.datax.plugin.writer.kudu11xwriter;
+
+/**
+ * @author daizihao
+ * @create 2020-08-31 14:17
+ **/
+public class Key {
+ public final static String KUDU_CONFIG = "kuduConfig";
+ public final static String KUDU_MASTER = "kudu.master_addresses";
+ public final static String KUDU_ADMIN_TIMEOUT = "timeout";
+ public final static String KUDU_SESSION_TIMEOUT = "sessionTimeout";
+
+ public final static String TABLE = "table";
+ public final static String PARTITION = "partition";
+ public final static String COLUMN = "column";
+
+ public static final String NAME = "name";
+ public static final String TYPE = "type";
+ public static final String INDEX = "index";
+ public static final String PRIMARYKEY = "primaryKey";
+ public static final String COMPRESSION = "compress";
+ public static final String COMMENT = "comment";
+ public final static String ENCODING = "encoding";
+
+
+
+ public static final String NUM_REPLICAS = "replicaCount";
+ public static final String HASH = "hash";
+ public static final String HASH_NUM = "number";
+
+ public static final String RANGE = "range";
+ public static final String LOWER = "lower";
+ public static final String UPPER = "upper";
+
+
+
+ public static final String TRUNCATE = "truncate";
+
+ public static final String INSERT_MODE = "writeMode";
+
+ public static final String WRITE_BATCH_SIZE = "batchSize";
+
+ public static final String MUTATION_BUFFER_SPACE = "bufferSize";
+ public static final String SKIP_FAIL = "skipFail";
+}
diff --git a/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Kudu11xHelper.java b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Kudu11xHelper.java
new file mode 100644
index 00000000..cf1b0f8f
--- /dev/null
+++ b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Kudu11xHelper.java
@@ -0,0 +1,369 @@
+package com.q1.datax.plugin.writer.kudu11xwriter;
+
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.fastjson.JSON;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Validate;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.*;
+import org.apache.kudu.shaded.org.checkerframework.checker.units.qual.K;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sun.rmi.runtime.Log;
+
+import java.nio.charset.Charset;
+import java.util.*;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * @author daizihao
+ * @create 2020-08-27 18:30
+ **/
+public class Kudu11xHelper {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Kudu11xHelper.class);
+
+ public static Map getKuduConfiguration(String kuduConfig) {
+ if (StringUtils.isBlank(kuduConfig)) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.REQUIRED_VALUE,
+ "Connection configuration information required.");
+ }
+ Map kConfiguration;
+ try {
+ kConfiguration = JSON.parseObject(kuduConfig, HashMap.class);
+ Validate.isTrue(kConfiguration != null, "kuduConfig is null!");
+ kConfiguration.put(Key.KUDU_ADMIN_TIMEOUT, kConfiguration.getOrDefault(Key.KUDU_ADMIN_TIMEOUT, Constant.ADMIN_TIMEOUTMS));
+ kConfiguration.put(Key.KUDU_SESSION_TIMEOUT, kConfiguration.getOrDefault(Key.KUDU_SESSION_TIMEOUT, Constant.SESSION_TIMEOUTMS));
+ } catch (Exception e) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.GET_KUDU_CONNECTION_ERROR, e);
+ }
+
+ return kConfiguration;
+ }
+
+ public static KuduClient getKuduClient(String kuduConfig) {
+ Map conf = Kudu11xHelper.getKuduConfiguration(kuduConfig);
+ KuduClient kuduClient = null;
+ try {
+ String masterAddress = (String) conf.get(Key.KUDU_MASTER);
+ kuduClient = new KuduClient.KuduClientBuilder(masterAddress)
+ .defaultAdminOperationTimeoutMs((Long) conf.get(Key.KUDU_ADMIN_TIMEOUT))
+ .defaultOperationTimeoutMs((Long) conf.get(Key.KUDU_SESSION_TIMEOUT))
+ .build();
+ } catch (Exception e) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.GET_KUDU_CONNECTION_ERROR, e);
+ }
+ return kuduClient;
+ }
+
+ public static KuduTable getKuduTable(Configuration configuration, KuduClient kuduClient) {
+ String tableName = configuration.getString(Key.TABLE);
+
+ KuduTable table = null;
+ try {
+ if (kuduClient.tableExists(tableName)) {
+ table = kuduClient.openTable(tableName);
+ } else {
+ synchronized (Kudu11xHelper.class) {
+ if (!kuduClient.tableExists(tableName)) {
+ Schema schema = Kudu11xHelper.getSchema(configuration);
+ CreateTableOptions tableOptions = new CreateTableOptions();
+
+ Kudu11xHelper.setTablePartition(configuration, tableOptions, schema);
+ //副本数
+ Integer numReplicas = configuration.getInt(Key.NUM_REPLICAS, 3);
+ tableOptions.setNumReplicas(numReplicas);
+ table = kuduClient.createTable(tableName, schema, tableOptions);
+ } else {
+ table = kuduClient.openTable(tableName);
+ }
+ }
+ }
+
+
+ } catch (Exception e) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.GET_KUDU_TABLE_ERROR, e);
+ }
+ return table;
+ }
+
+ public static void createTable(Configuration configuration) {
+ String tableName = configuration.getString(Key.TABLE);
+ String kuduConfig = configuration.getString(Key.KUDU_CONFIG);
+ KuduClient kuduClient = Kudu11xHelper.getKuduClient(kuduConfig);
+ try {
+ Schema schema = Kudu11xHelper.getSchema(configuration);
+ CreateTableOptions tableOptions = new CreateTableOptions();
+
+ Kudu11xHelper.setTablePartition(configuration, tableOptions, schema);
+ //副本数
+ Integer numReplicas = configuration.getInt(Key.NUM_REPLICAS, 3);
+ tableOptions.setNumReplicas(numReplicas);
+ kuduClient.createTable(tableName, schema, tableOptions);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.GREATE_KUDU_TABLE_ERROR, e);
+ } finally {
+ AtomicInteger i = new AtomicInteger(10);
+ while (i.get() > 0) {
+ try {
+ if (kuduClient.isCreateTableDone(tableName)) {
+ Kudu11xHelper.closeClient(kuduClient);
+ LOG.info("Table " + tableName + " is created!");
+ break;
+ }
+ i.decrementAndGet();
+ LOG.error("timeout!");
+ } catch (KuduException e) {
+ LOG.info("Wait for the table to be created..... " + i);
+ try {
+ Thread.sleep(100L);
+ } catch (InterruptedException ex) {
+ ex.printStackTrace();
+ }
+ i.decrementAndGet();
+ }
+ }
+ try {
+ if (kuduClient != null) {
+ kuduClient.close();
+ }
+ } catch (KuduException e) {
+ LOG.info("Kudu client has been shut down!");
+ }
+ }
+ }
+
+ public static ThreadPoolExecutor createRowAddThreadPool(int coreSize) {
+ return new ThreadPoolExecutor(coreSize,
+ coreSize,
+ 60L,
+ TimeUnit.SECONDS,
+ new SynchronousQueue(),
+ new ThreadFactory() {
+ private final ThreadGroup group = System.getSecurityManager() == null ? Thread.currentThread().getThreadGroup() : System.getSecurityManager().getThreadGroup();
+ private final AtomicInteger threadNumber = new AtomicInteger(1);
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(group, r,
+ "pool-kudu_rows_add-thread-" + threadNumber.getAndIncrement(),
+ 0);
+ if (t.isDaemon())
+ t.setDaemon(false);
+ if (t.getPriority() != Thread.NORM_PRIORITY)
+ t.setPriority(Thread.NORM_PRIORITY);
+ return t;
+ }
+ }, new ThreadPoolExecutor.CallerRunsPolicy());
+ }
+
+ public static List> getColumnLists(List columns) {
+ int quota = 8;
+ int num = (columns.size() - 1) / quota + 1;
+ int gap = columns.size() / num;
+ List> columnLists = new ArrayList<>(num);
+ for (int j = 0; j < num - 1; j++) {
+ List destList = new ArrayList<>(columns.subList(j * gap, (j + 1) * gap));
+ columnLists.add(destList);
+ }
+ List destList = new ArrayList<>(columns.subList(gap * (num - 1), columns.size()));
+ columnLists.add(destList);
+ return columnLists;
+ }
+
+ public static boolean isTableExists(Configuration configuration) {
+ String tableName = configuration.getString(Key.TABLE);
+ String kuduConfig = configuration.getString(Key.KUDU_CONFIG);
+ KuduClient kuduClient = Kudu11xHelper.getKuduClient(kuduConfig);
+ try {
+ return kuduClient.tableExists(tableName);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.GET_KUDU_CONNECTION_ERROR, e);
+ } finally {
+ Kudu11xHelper.closeClient(kuduClient);
+ }
+ }
+
+ public static void closeClient(KuduClient kuduClient) {
+ try {
+ if (kuduClient != null) {
+ kuduClient.close();
+ }
+ } catch (KuduException e) {
+ LOG.warn("The \"kudu client\" was not stopped gracefully. !");
+
+ }
+
+ }
+
+ public static Schema getSchema(Configuration configuration) {
+ List columns = configuration.getListConfiguration(Key.COLUMN);
+ List columnSchemas = new ArrayList<>();
+ Schema schema = null;
+ if (columns == null || columns.isEmpty()) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.REQUIRED_VALUE, "column is not defined，eg：column:[{\"name\": \"cf0:column0\",\"type\": \"string\"},{\"name\": \"cf1:column1\",\"type\": \"long\"}]");
+ }
+ try {
+ for (Configuration column : columns) {
+
+ String type = "BIGINT".equals(column.getNecessaryValue(Key.TYPE, Kudu11xWriterErrorcode.REQUIRED_VALUE).toUpperCase()) ||
+ "LONG".equals(column.getNecessaryValue(Key.TYPE, Kudu11xWriterErrorcode.REQUIRED_VALUE).toUpperCase()) ?
+ "INT64" : "INT".equals(column.getNecessaryValue(Key.TYPE, Kudu11xWriterErrorcode.REQUIRED_VALUE).toUpperCase()) ?
+ "INT32" : column.getNecessaryValue(Key.TYPE, Kudu11xWriterErrorcode.REQUIRED_VALUE).toUpperCase();
+ String name = column.getNecessaryValue(Key.NAME, Kudu11xWriterErrorcode.REQUIRED_VALUE);
+ Boolean key = column.getBool(Key.PRIMARYKEY, false);
+ String encoding = column.getString(Key.ENCODING, Constant.ENCODING).toUpperCase();
+ String compression = column.getString(Key.COMPRESSION, Constant.COMPRESSION).toUpperCase();
+ String comment = column.getString(Key.COMMENT, "");
+
+ columnSchemas.add(new ColumnSchema.ColumnSchemaBuilder(name, Type.getTypeForName(type))
+ .key(key)
+ .encoding(ColumnSchema.Encoding.valueOf(encoding))
+ .compressionAlgorithm(ColumnSchema.CompressionAlgorithm.valueOf(compression))
+ .comment(comment)
+ .build());
+ }
+ schema = new Schema(columnSchemas);
+ } catch (Exception e) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.REQUIRED_VALUE, e);
+ }
+ return schema;
+ }
+
+ public static Integer getPrimaryKeyIndexUntil(List columns) {
+ int i = 0;
+ while (i < columns.size()) {
+ Configuration col = columns.get(i);
+ if (!col.getBool(Key.PRIMARYKEY, false)) {
+ break;
+ }
+ i++;
+ }
+ return i;
+ }
+
+ public static void setTablePartition(Configuration configuration,
+ CreateTableOptions tableOptions,
+ Schema schema) {
+ Configuration partition = configuration.getConfiguration(Key.PARTITION);
+ if (partition == null) {
+ ColumnSchema columnSchema = schema.getColumns().get(0);
+ tableOptions.addHashPartitions(Collections.singletonList(columnSchema.getName()), 3);
+ return;
+ }
+ //range分区
+ Configuration range = partition.getConfiguration(Key.RANGE);
+ if (range != null) {
+ List rangeColums = new ArrayList<>(range.getKeys());
+ tableOptions.setRangePartitionColumns(rangeColums);
+ for (String rangeColum : rangeColums) {
+ List lowerAndUppers = range.getListConfiguration(rangeColum);
+ for (Configuration lowerAndUpper : lowerAndUppers) {
+ PartialRow lower = schema.newPartialRow();
+ lower.addString(rangeColum, lowerAndUpper.getNecessaryValue(Key.LOWER, Kudu11xWriterErrorcode.REQUIRED_VALUE));
+ PartialRow upper = schema.newPartialRow();
+ upper.addString(rangeColum, lowerAndUpper.getNecessaryValue(Key.UPPER, Kudu11xWriterErrorcode.REQUIRED_VALUE));
+ tableOptions.addRangePartition(lower, upper);
+ }
+ }
+ LOG.info("Set range partition complete!");
+ }
+
+ // 设置Hash分区
+ Configuration hash = partition.getConfiguration(Key.HASH);
+ if (hash != null) {
+ List hashColums = hash.getList(Key.COLUMN, String.class);
+ Integer hashPartitionNum = configuration.getInt(Key.HASH_NUM, 3);
+ tableOptions.addHashPartitions(hashColums, hashPartitionNum);
+ LOG.info("Set hash partition complete!");
+ }
+ }
+
+ public static void validateParameter(Configuration configuration) {
+ LOG.info("Start validating parameters！");
+ configuration.getNecessaryValue(Key.KUDU_CONFIG, Kudu11xWriterErrorcode.REQUIRED_VALUE);
+ configuration.getNecessaryValue(Key.TABLE, Kudu11xWriterErrorcode.REQUIRED_VALUE);
+ String encoding = configuration.getString(Key.ENCODING, Constant.DEFAULT_ENCODING);
+ if (!Charset.isSupported(encoding)) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.ILLEGAL_VALUE,
+ String.format("Encoding is not supported:[%s] .", encoding));
+ }
+ configuration.set(Key.ENCODING, encoding);
+ String insertMode = configuration.getString(Key.INSERT_MODE, Constant.INSERT_MODE);
+ try {
+ InsertModeType.getByTypeName(insertMode);
+ } catch (Exception e) {
+ insertMode = Constant.INSERT_MODE;
+ }
+ configuration.set(Key.INSERT_MODE, insertMode);
+
+ Long writeBufferSize = configuration.getLong(Key.WRITE_BATCH_SIZE, Constant.DEFAULT_WRITE_BATCH_SIZE);
+ configuration.set(Key.WRITE_BATCH_SIZE, writeBufferSize);
+
+ Long mutationBufferSpace = configuration.getLong(Key.MUTATION_BUFFER_SPACE, Constant.DEFAULT_MUTATION_BUFFER_SPACE);
+ configuration.set(Key.MUTATION_BUFFER_SPACE, mutationBufferSpace);
+
+ Boolean isSkipFail = configuration.getBool(Key.SKIP_FAIL, false);
+ configuration.set(Key.SKIP_FAIL, isSkipFail);
+ List columns = configuration.getListConfiguration(Key.COLUMN);
+ List goalColumns = new ArrayList<>();
+ //column参数验证
+ int indexFlag = 0;
+ boolean primaryKey = true;
+ int primaryKeyFlag = 0;
+ for (int i = 0; i < columns.size(); i++) {
+ Configuration col = columns.get(i);
+ String index = col.getString(Key.INDEX);
+ if (index == null) {
+ index = String.valueOf(i);
+ col.set(Key.INDEX, index);
+ indexFlag++;
+ }
+ if(primaryKey != col.getBool(Key.PRIMARYKEY, false)){
+ primaryKey = col.getBool(Key.PRIMARYKEY, false);
+ primaryKeyFlag++;
+ }
+ goalColumns.add(col);
+ }
+ if (indexFlag != 0 && indexFlag != columns.size()) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.ILLEGAL_VALUE,
+ "\"index\" either has values for all of them, or all of them are null!");
+ }
+ if (primaryKeyFlag > 1){
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.ILLEGAL_VALUE,
+ "\"primaryKey\" must be written in the front！");
+ }
+ configuration.set(Key.COLUMN, goalColumns);
+// LOG.info("------------------------------------");
+// LOG.info(configuration.toString());
+// LOG.info("------------------------------------");
+ LOG.info("validate parameter complete！");
+ }
+
+ public static void truncateTable(Configuration configuration) {
+ String kuduConfig = configuration.getString(Key.KUDU_CONFIG);
+ String userTable = configuration.getString(Key.TABLE);
+ LOG.info(String.format("Because you have configured truncate is true,KuduWriter begins to truncate table %s .", userTable));
+ KuduClient kuduClient = Kudu11xHelper.getKuduClient(kuduConfig);
+
+ try {
+ if (kuduClient.tableExists(userTable)) {
+ kuduClient.deleteTable(userTable);
+ LOG.info(String.format("table %s has been deleted.", userTable));
+ }
+ } catch (KuduException e) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.DELETE_KUDU_ERROR, e);
+ } finally {
+ Kudu11xHelper.closeClient(kuduClient);
+ }
+
+ }
+}
diff --git a/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Kudu11xWriter.java b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Kudu11xWriter.java
new file mode 100644
index 00000000..83620f43
--- /dev/null
+++ b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Kudu11xWriter.java
@@ -0,0 +1,85 @@
+package com.q1.datax.plugin.writer.kudu11xwriter;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+import com.alibaba.datax.common.spi.Writer;
+import com.alibaba.datax.common.util.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author daizihao
+ * @create 2020-08-27 16:58
+ **/
+public class Kudu11xWriter extends Writer {
+ public static class Job extends Writer.Job{
+ private static final Logger LOG = LoggerFactory.getLogger(Job.class);
+ private Configuration config = null;
+ @Override
+ public void init() {
+ this.config = this.getPluginJobConf();
+ Kudu11xHelper.validateParameter(this.config);
+ }
+
+ @Override
+ public void prepare() {
+ Boolean truncate = config.getBool(Key.TRUNCATE,false);
+ if(truncate){
+ Kudu11xHelper.truncateTable(this.config);
+ }
+
+ if (!Kudu11xHelper.isTableExists(config)){
+ Kudu11xHelper.createTable(config);
+ }
+ }
+
+ @Override
+ public List split(int i) {
+ List splitResultConfigs = new ArrayList<>();
+ for (int j = 0; j < i; j++) {
+ splitResultConfigs.add(config.clone());
+ }
+
+ return splitResultConfigs;
+ }
+
+
+
+ @Override
+ public void destroy() {
+
+ }
+ }
+
+ public static class Task extends Writer.Task{
+ private Configuration taskConfig;
+ private KuduWriterTask kuduTaskProxy;
+ private static final Logger LOG = LoggerFactory.getLogger(Job.class);
+ @Override
+ public void init() {
+ this.taskConfig = super.getPluginJobConf();
+ this.kuduTaskProxy = new KuduWriterTask(this.taskConfig);
+ }
+ @Override
+ public void startWrite(RecordReceiver lineReceiver) {
+ this.kuduTaskProxy.startWriter(lineReceiver,super.getTaskPluginCollector());
+ }
+
+
+ @Override
+ public void destroy() {
+ try {
+ if (kuduTaskProxy.session != null) {
+ kuduTaskProxy.session.close();
+ }
+ }catch (Exception e){
+ LOG.warn("The \"kudu session\" was not stopped gracefully !");
+ }
+ Kudu11xHelper.closeClient(kuduTaskProxy.kuduClient);
+
+ }
+ }
+}
diff --git a/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Kudu11xWriterErrorcode.java b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Kudu11xWriterErrorcode.java
new file mode 100644
index 00000000..d46bcea3
--- /dev/null
+++ b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Kudu11xWriterErrorcode.java
@@ -0,0 +1,39 @@
+package com.q1.datax.plugin.writer.kudu11xwriter;
+
+import com.alibaba.datax.common.spi.ErrorCode;
+
+/**
+ * @author daizihao
+ * @create 2020-08-27 19:25
+ **/
+public enum Kudu11xWriterErrorcode implements ErrorCode {
+ REQUIRED_VALUE("Kuduwriter-00", "You are missing a required parameter value."),
+ ILLEGAL_VALUE("Kuduwriter-01", "You fill in the parameter values are not legitimate."),
+ GET_KUDU_CONNECTION_ERROR("Kuduwriter-02", "Error getting Kudu connection."),
+ GET_KUDU_TABLE_ERROR("Kuduwriter-03", "Error getting Kudu table."),
+ CLOSE_KUDU_CONNECTION_ERROR("Kuduwriter-04", "Error closing Kudu connection."),
+ CLOSE_KUDU_SESSION_ERROR("Kuduwriter-06", "Error closing Kudu table connection."),
+ PUT_KUDU_ERROR("Kuduwriter-07", "IO exception occurred when writing to Kudu."),
+ DELETE_KUDU_ERROR("Kuduwriter-08", "An exception occurred while delete Kudu table."),
+ GREATE_KUDU_TABLE_ERROR("Kuduwriter-09", "Error creating Kudu table."),
+ PARAMETER_NUM_ERROR("Kuduwriter-10","The number of parameters does not match.")
+ ;
+
+ private final String code;
+ private final String description;
+
+
+ Kudu11xWriterErrorcode(String code, String description) {
+ this.code = code;
+ this.description = description;
+ }
+ @Override
+ public String getCode() {
+ return code;
+ }
+
+ @Override
+ public String getDescription() {
+ return description;
+ }
+}
diff --git a/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/KuduWriterTask.java b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/KuduWriterTask.java
new file mode 100644
index 00000000..bff3509f
--- /dev/null
+++ b/kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/KuduWriterTask.java
@@ -0,0 +1,216 @@
+package com.q1.datax.plugin.writer.kudu11xwriter;
+
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordReceiver;
+import com.alibaba.datax.common.plugin.TaskPluginCollector;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.common.util.RetryUtil;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kudu.client.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
+
+/**
+ * @author daizihao
+ * @create 2020-08-31 16:55
+ **/
+public class KuduWriterTask {
+ private final static Logger LOG = LoggerFactory.getLogger(KuduWriterTask.class);
+
+ private List columns;
+ private List> columnLists;
+ private ThreadPoolExecutor pool;
+ private String encoding;
+ private Double batchSize;
+ private Boolean isUpsert;
+ private Boolean isSkipFail;
+ public KuduClient kuduClient;
+ public KuduSession session;
+ private KuduTable table;
+ private Integer primaryKeyIndexUntil;
+
+ private final Object lock = new Object();
+
+ public KuduWriterTask(Configuration configuration) {
+ columns = configuration.getListConfiguration(Key.COLUMN);
+ columnLists = Kudu11xHelper.getColumnLists(columns);
+ pool = Kudu11xHelper.createRowAddThreadPool(columnLists.size());
+
+ this.encoding = configuration.getString(Key.ENCODING);
+ this.batchSize = configuration.getDouble(Key.WRITE_BATCH_SIZE);
+ this.isUpsert = !configuration.getString(Key.INSERT_MODE).equalsIgnoreCase("insert");
+ this.isSkipFail = configuration.getBool(Key.SKIP_FAIL);
+ long mutationBufferSpace = configuration.getLong(Key.MUTATION_BUFFER_SPACE);
+
+ this.kuduClient = Kudu11xHelper.getKuduClient(configuration.getString(Key.KUDU_CONFIG));
+ this.table = Kudu11xHelper.getKuduTable(configuration, kuduClient);
+ this.session = kuduClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ session.setMutationBufferSpace((int) mutationBufferSpace);
+ this.primaryKeyIndexUntil = Kudu11xHelper.getPrimaryKeyIndexUntil(columns);
+// tableName = configuration.getString(Key.TABLE);
+ }
+
+ public void startWriter(RecordReceiver lineReceiver, TaskPluginCollector taskPluginCollector) {
+ LOG.info("kuduwriter began to write!");
+ Record record;
+ LongAdder counter = new LongAdder();
+ try {
+ while ((record = lineReceiver.getFromReader()) != null) {
+ if (record.getColumnNumber() != columns.size()) {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.PARAMETER_NUM_ERROR, " number of record fields:" + record.getColumnNumber() + " number of configuration fields:" + columns.size());
+ }
+ boolean isDirtyRecord = false;
+
+
+ for (int i = 0; i < primaryKeyIndexUntil && !isDirtyRecord; i++) {
+ Column column = record.getColumn(i);
+ isDirtyRecord = StringUtils.isBlank(column.asString());
+ }
+
+ if (isDirtyRecord) {
+ taskPluginCollector.collectDirtyRecord(record, "primarykey field is null");
+ continue;
+ }
+
+ CountDownLatch countDownLatch = new CountDownLatch(columnLists.size());
+ Upsert upsert = table.newUpsert();
+ Insert insert = table.newInsert();
+ PartialRow row;
+ if (isUpsert) {
+ //覆盖更新
+ row = upsert.getRow();
+ } else {
+ //增量更新
+ row = insert.getRow();
+ }
+ List> futures = new ArrayList<>();
+ for (List columnList : columnLists) {
+ Record finalRecord = record;
+ Future future = pool.submit(() -> {
+ try {
+ for (Configuration col : columnList) {
+ String name = col.getString(Key.NAME);
+ ColumnType type = ColumnType.getByTypeName(col.getString(Key.TYPE, "string"));
+ Column column = finalRecord.getColumn(col.getInt(Key.INDEX));
+ String rawData = column.asString();
+ if (rawData == null) {
+ synchronized (lock) {
+ row.setNull(name);
+ }
+ continue;
+ }
+ switch (type) {
+ case INT:
+ synchronized (lock) {
+ row.addInt(name, Integer.parseInt(rawData));
+ }
+ break;
+ case LONG:
+ case BIGINT:
+ synchronized (lock) {
+ row.addLong(name, Long.parseLong(rawData));
+ }
+ break;
+ case FLOAT:
+ synchronized (lock) {
+ row.addFloat(name, Float.parseFloat(rawData));
+ }
+ break;
+ case DOUBLE:
+ synchronized (lock) {
+ row.addDouble(name, Double.parseDouble(rawData));
+ }
+ break;
+ case BOOLEAN:
+ synchronized (lock) {
+ row.addBoolean(name, Boolean.getBoolean(rawData));
+ }
+ break;
+ case STRING:
+ default:
+ synchronized (lock) {
+ row.addString(name, rawData);
+ }
+ }
+ }
+ } finally {
+ countDownLatch.countDown();
+ }
+ });
+ futures.add(future);
+ }
+ countDownLatch.await();
+ for (Future future : futures) {
+ future.get();
+ }
+ try {
+ RetryUtil.executeWithRetry(() -> {
+ if (isUpsert) {
+ //覆盖更新
+ session.apply(upsert);
+ } else {
+ //增量更新
+ session.apply(insert);
+ }
+ //flush
+ if (counter.longValue() > (batchSize * 0.8)) {
+ session.flush();
+ counter.reset();
+ }
+ counter.increment();
+ return true;
+ }, 5, 500L, true);
+
+ } catch (Exception e) {
+ LOG.error("Record Write Failure!", e);
+ if (isSkipFail) {
+ LOG.warn("Since you have configured \"skipFail\" to be true, this record will be skipped !");
+ taskPluginCollector.collectDirtyRecord(record, e.getMessage());
+ } else {
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.PUT_KUDU_ERROR, e.getMessage());
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("write failure! the task will exit!");
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.PUT_KUDU_ERROR, e.getMessage());
+ }
+ AtomicInteger i = new AtomicInteger(10);
+ try {
+ while (i.get() > 0) {
+ if (session.hasPendingOperations()) {
+ session.flush();
+ break;
+ }
+ Thread.sleep(20L);
+ i.decrementAndGet();
+ }
+ } catch (Exception e) {
+ LOG.info("Waiting for data to be written to kudu...... " + i + "s");
+
+ } finally {
+ try {
+ pool.shutdown();
+ //强制刷写
+ session.flush();
+ } catch (KuduException e) {
+ LOG.error("kuduwriter flush error! The results may be incomplete！");
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.PUT_KUDU_ERROR, e.getMessage());
+ }
+ }
+
+ }
+
+
+}
diff --git a/kuduwriter/src/main/java/com/q1/kudu/conf/KuduConfig.java b/kuduwriter/src/main/java/com/q1/kudu/conf/KuduConfig.java
new file mode 100644
index 00000000..f1499a0f
--- /dev/null
+++ b/kuduwriter/src/main/java/com/q1/kudu/conf/KuduConfig.java
@@ -0,0 +1,9 @@
+package com.q1.kudu.conf;
+
+/**
+ * @author daizihao
+ * @create 2020-09-16 11:39
+ **/
+public class KuduConfig {
+
+}
diff --git a/kuduwriter/src/main/resources/plugin.json b/kuduwriter/src/main/resources/plugin.json
new file mode 100644
index 00000000..f60dc825
--- /dev/null
+++ b/kuduwriter/src/main/resources/plugin.json
@@ -0,0 +1,7 @@
+{
+ "name": "kuduwriter",
+ "class": "com.q1.datax.plugin.writer.kudu11xwriter.Kudu11xWriter",
+ "description": "use put: prod. mechanism: use kudu java api put data.",
+ "developer": "com.q1.daizihao"
+}
+
diff --git a/kuduwriter/src/main/resources/plugin_job_template.json b/kuduwriter/src/main/resources/plugin_job_template.json
new file mode 100644
index 00000000..3edc6c39
--- /dev/null
+++ b/kuduwriter/src/main/resources/plugin_job_template.json
@@ -0,0 +1,59 @@
+{
+ "name": "kuduwriter",
+ "parameter": {
+ "kuduConfig": {
+ "kudu.master_addresses": "***",
+ "timeout": 60000,
+ "sessionTimeout": 60000
+
+ },
+ "table": "",
+ "replicaCount": 3,
+ "truncate": false,
+ "writeMode": "upsert",
+ "partition": {
+ "range": {
+ "column1": [
+ {
+ "lower": "2020-08-25",
+ "upper": "2020-08-26"
+ },
+ {
+ "lower": "2020-08-26",
+ "upper": "2020-08-27"
+ },
+ {
+ "lower": "2020-08-27",
+ "upper": "2020-08-28"
+ }
+ ]
+ },
+ "hash": {
+ "column": [
+ "column1"
+ ],
+ "number": 3
+ }
+ },
+ "column": [
+ {
+ "index": 0,
+ "name": "c1",
+ "type": "string",
+ "primaryKey": true
+ },
+ {
+ "index": 1,
+ "name": "c2",
+ "type": "string",
+ "compress": "DEFAULT_COMPRESSION",
+ "encoding": "AUTO_ENCODING",
+ "comment": "注解xxxx"
+ }
+ ],
+ "batchSize": 1024,
+ "bufferSize": 2048,
+ "skipFail": false,
+ "encoding": "UTF-8"
+ }
+}
\ No newline at end of file
diff --git a/kuduwriter/src/test/java/com/dai/test.java b/kuduwriter/src/test/java/com/dai/test.java
new file mode 100644
index 00000000..5fd17beb
--- /dev/null
+++ b/kuduwriter/src/test/java/com/dai/test.java
@@ -0,0 +1,40 @@
+package com.dai;
+
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.util.RetryUtil;
+import com.q1.datax.plugin.writer.kudu11xwriter.*;
+import static org.apache.kudu.client.AsyncKuduClient.LOG;
+
+/**
+ * @author daizihao
+ * @create 2020-08-28 11:03
+ **/
+public class test {
+ static boolean isSkipFail;
+
+
+ public static void main(String[] args) {
+ try {
+ while (true) {
+ try {
+ RetryUtil.executeWithRetry(()->{
+ throw new RuntimeException();
+ },5,1000L,true);
+
+ } catch (Exception e) {
+ LOG.error("Data write failed!", e);
+ System.out.println(isSkipFail);
+ if (isSkipFail) {
+ LOG.warn("Because you have configured skipFail is true,this data will be skipped!");
+ }else {
+ System.out.println("异常抛出");
+ throw e;
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("write failed! the task will exit!");
+ throw DataXException.asDataXException(Kudu11xWriterErrorcode.PUT_KUDU_ERROR, e);
+ }
+ }
+}
diff --git a/mongodbreader/doc/mongodbreader.md b/mongodbreader/doc/mongodbreader.md
index 3535d5b7..b61493e6 100644
--- a/mongodbreader/doc/mongodbreader.md
+++ b/mongodbreader/doc/mongodbreader.md
@@ -8,7 +8,7 @@ MongoDBReader 插件利用 MongoDB 的java客户端MongoClient进行MongoDB的
MongoDBReader通过Datax框架从MongoDB并行的读取数据，通过主控的JOB程序按照指定的规则对MongoDB中的数据进行分片，并行读取，然后将MongoDB支持的类型通过逐一判断转换成Datax支持的类型。
#### 3 功能说明
-* 该示例从ODPS读一份数据到MongoDB。
+* 该示例从MongoDB读一份数据到ODPS。
{
"job": {
@@ -132,6 +132,7 @@ MongoDBReader通过Datax框架从MongoDB并行的读取数据，通过主控的J
* name：Column的名字。【必填】
* type：Column的类型。【选填】
* splitter：因为MongoDB支持数组类型，但是Datax框架本身不支持数组类型，所以mongoDB读出来的数组类型要通过这个分隔符合并成字符串。【选填】
+* query: MongoDB的额外查询条件。【选填】
#### 5 类型转换
@@ -146,4 +147,4 @@ MongoDBReader通过Datax框架从MongoDB并行的读取数据，通过主控的J
#### 6 性能报告
-#### 7 测试报告
\ No newline at end of file
+#### 7 测试报告
diff --git a/mongodbwriter/doc/mongodbwriter.md b/mongodbwriter/doc/mongodbwriter.md
index e30008db..74de8a0a 100644
--- a/mongodbwriter/doc/mongodbwriter.md
+++ b/mongodbwriter/doc/mongodbwriter.md
@@ -139,7 +139,7 @@ MongoDBWriter通过Datax框架获取Reader生成的数据，然后将Datax支持
* splitter：特殊分隔符，当且仅当要处理的字符串要用分隔符分隔为字符数组时，才使用这个参数，通过这个参数指定的分隔符，将字符串分隔存储到MongoDB的数组中。【选填】
* upsertInfo：指定了传输数据时更新的信息。【选填】
* isUpsert：当设置为true时，表示针对相同的upsertKey做更新操作。【选填】
-* upsertKey：upsertKey指定了没行记录的业务主键。用来做更新时使用。【选填】
+* upsertKey：upsertKey指定了每行记录的业务主键。用来做更新时使用。【选填】
#### 5 类型转换
@@ -154,4 +154,4 @@ MongoDBWriter通过Datax框架获取Reader生成的数据，然后将Datax支持
#### 6 性能报告
-#### 7 测试报告
\ No newline at end of file
+#### 7 测试报告
diff --git a/mysqlreader/pom.xml b/mysqlreader/pom.xml
index 08183272..621326ae 100755
--- a/mysqlreader/pom.xml
+++ b/mysqlreader/pom.xml
@@ -40,7 +40,7 @@
mysql
mysql-connector-java
- 5.1.34
+ ${mysql.driver.version}
diff --git a/mysqlwriter/doc/mysqlwriter.md b/mysqlwriter/doc/mysqlwriter.md
index f6abd242..5368775c 100644
--- a/mysqlwriter/doc/mysqlwriter.md
+++ b/mysqlwriter/doc/mysqlwriter.md
@@ -147,7 +147,7 @@ MysqlWriter 通过 DataX 框架获取 Reader 生成的协议数据，根据你
* **column**
- * 描述：目的表需要写入数据的字段,字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。如果要依次写入全部列，使用*表示, 例如: "column": ["*"]。
+ * 描述：目的表需要写入数据的字段,字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。如果要依次写入全部列，使用`*`表示, 例如: `"column": ["*"]`。
**column配置项必须指定，不能留空！**
diff --git a/mysqlwriter/pom.xml b/mysqlwriter/pom.xml
index 11618022..1c3891f5 100755
--- a/mysqlwriter/pom.xml
+++ b/mysqlwriter/pom.xml
@@ -40,7 +40,7 @@
mysql
mysql-connector-java
- 5.1.34
+ ${mysql.driver.version}
diff --git a/oceanbasev10reader/pom.xml b/oceanbasev10reader/pom.xml
new file mode 100644
index 00000000..49477241
--- /dev/null
+++ b/oceanbasev10reader/pom.xml
@@ -0,0 +1,97 @@
+
+
+
+ datax-all
+ com.alibaba.datax
+ 0.0.1-SNAPSHOT
+
+ 4.0.0
+ oceanbasev10reader
+ com.alibaba.datax
+ 0.0.1-SNAPSHOT
+ jar
+
+
+
+ com.alibaba.datax
+ datax-common
+ ${datax-project-version}
+
+
+ slf4j-log4j12
+ org.slf4j
+
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+ ch.qos.logback
+ logback-classic
+
+
+ com.alibaba.datax
+ plugin-rdbms-util
+ ${datax-project-version}
+
+
+ mysql
+ mysql-connector-java
+ 5.1.40
+
+
+ log4j
+ log4j
+ 1.2.16
+
+
+ junit
+ junit
+ 4.11
+ test
+
+
+
+
+
+
+ src/main/java
+
+ **/*.properties
+
+
+
+
+
+
+ maven-compiler-plugin
+
+ ${jdk-version}
+ ${jdk-version}
+ ${project-sourceEncoding}
+
+
+
+
+ maven-assembly-plugin
+
+
+ src/main/assembly/package.xml
+
+ datax
+
+
+
+ dwzip
+ package
+
+ single
+
+
+
+
+
+
+
diff --git a/oceanbasev10reader/src/main/assembly/package.xml b/oceanbasev10reader/src/main/assembly/package.xml
new file mode 100644
index 00000000..c1db32a9
--- /dev/null
+++ b/oceanbasev10reader/src/main/assembly/package.xml
@@ -0,0 +1,42 @@
+
+
+
+ dir
+
+ false
+
+
+ src/main/resources
+
+ plugin.json
+ plugin_job_template.json
+
+ plugin/reader/oceanbasev10reader
+
+
+ target/
+
+ oceanbasev10reader-0.0.1-SNAPSHOT.jar
+
+ plugin/reader/oceanbasev10reader
+
+
+ src/main/libs/
+
+ *.jar
+
+ plugin/reader/oceanbasev10reader/libs
+
+
+
+
+
+ false
+ plugin/reader/oceanbasev10reader/libs
+ runtime
+
+
+
diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/Config.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/Config.java
new file mode 100644
index 00000000..ca803c49
--- /dev/null
+++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/Config.java
@@ -0,0 +1,16 @@
+package com.alibaba.datax.plugin.reader.oceanbasev10reader;
+
+public interface Config {
+ // queryTimeoutSecond
+ String QUERY_TIMEOUT_SECOND = "memstoreCheckIntervalSecond";
+
+ int DEFAULT_QUERY_TIMEOUT_SECOND = 60 * 60 * 48;// 2天
+
+ // readBatchSize
+ String READ_BATCH_SIZE = "readBatchSize";
+
+ int DEFAULT_READ_BATCH_SIZE = 100000;// 10万
+
+ String RETRY_LIMIT = "retryLimit";
+ int DEFAULT_RETRY_LIMIT = 10;
+}
diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java
new file mode 100644
index 00000000..0a4934a1
--- /dev/null
+++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java
@@ -0,0 +1,127 @@
+package com.alibaba.datax.plugin.reader.oceanbasev10reader;
+
+import java.sql.Connection;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.datax.common.plugin.RecordSender;
+import com.alibaba.datax.common.spi.Reader;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.rdbms.reader.Constant;
+import com.alibaba.datax.plugin.rdbms.reader.Key;
+import com.alibaba.datax.plugin.rdbms.util.DBUtil;
+import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
+import com.alibaba.datax.plugin.reader.oceanbasev10reader.ext.ReaderJob;
+import com.alibaba.datax.plugin.reader.oceanbasev10reader.ext.ReaderTask;
+import com.alibaba.datax.plugin.reader.oceanbasev10reader.util.ObReaderUtils;
+
+public class OceanBaseReader extends Reader {
+
+ public static class Job extends Reader.Job {
+ private Configuration originalConfig = null;
+ private ReaderJob readerJob;
+ private static final Logger LOG = LoggerFactory.getLogger(Task.class);
+
+ @Override
+ public void init() {
+ this.originalConfig = super.getPluginJobConf();
+
+ Integer userConfigedFetchSize = this.originalConfig.getInt(Constant.FETCH_SIZE);
+ if (userConfigedFetchSize != null) {
+ LOG.warn("The [fetchSize] is not recognized, please use readBatchSize instead.");
+ }
+
+ this.originalConfig.set(Constant.FETCH_SIZE, Integer.MIN_VALUE);
+
+ setDatabaseType(originalConfig);
+
+ this.readerJob = new ReaderJob();
+ this.readerJob.init(this.originalConfig);
+ }
+
+ @Override
+ public void preCheck() {
+ init();
+ this.readerJob.preCheck(this.originalConfig, ObReaderUtils.DATABASE_TYPE);
+
+ }
+
+ @Override
+ public List split(int adviceNumber) {
+ return this.readerJob.split(this.originalConfig, adviceNumber);
+ }
+
+ @Override
+ public void post() {
+ this.readerJob.post(this.originalConfig);
+ }
+
+ @Override
+ public void destroy() {
+ this.readerJob.destroy(this.originalConfig);
+ }
+
+ private void setDatabaseType(Configuration config) {
+ String username = config.getString(Key.USERNAME);
+ String password = config.getString(Key.PASSWORD);
+ List
+- DataX开源用户交流群2
+ - 
+
+- DataX开源用户交流群3
+ - 
+
+- DataX开源用户交流群4
+ - 
+
+- DataX开源用户交流群5
+ - 
+
+- DataX开源用户交流群6
+ - 
diff --git a/clickhousewriter/pom.xml b/clickhousewriter/pom.xml
new file mode 100644
index 00000000..76c5fb1f
--- /dev/null
+++ b/clickhousewriter/pom.xml
@@ -0,0 +1,88 @@
+
+