From 2f8cc74b917b77fdbea297eb615eed1f99ac4f88 Mon Sep 17 00:00:00 2001 From: mazhenlin Date: Fri, 11 Oct 2019 16:30:56 +0800 Subject: [PATCH 1/4] cassandra plugins --- README.md | 1 + cassandrareader/doc/cassandrareader.md | 217 +++++++ cassandrareader/pom.xml | 133 ++++ cassandrareader/src/main/assembly/package.xml | 35 + .../cassandrareader/CassandraReader.java | 123 ++++ .../CassandraReaderErrorCode.java | 32 + .../CassandraReaderHelper.java | 607 ++++++++++++++++++ .../plugin/reader/cassandrareader/Key.java | 39 ++ .../cassandrareader/LocalStrings.properties | 1 + .../LocalStrings_en_US.properties | 0 .../LocalStrings_ja_JP.properties | 1 + .../LocalStrings_zh_CN.properties | 1 + .../LocalStrings_zh_HK.properties | 1 + .../LocalStrings_zh_TW.properties | 1 + .../src/main/resources/plugin.json | 6 + .../main/resources/plugin_job_template.json | 15 + cassandrawriter/doc/cassandrawriter.md | 227 +++++++ cassandrawriter/pom.xml | 125 ++++ cassandrawriter/src/main/assembly/package.xml | 35 + .../cassandrawriter/CassandraWriter.java | 242 +++++++ .../CassandraWriterErrorCode.java | 35 + .../CassandraWriterHelper.java | 351 ++++++++++ .../plugin/writer/cassandrawriter/Key.java | 43 ++ .../cassandrawriter/LocalStrings.properties | 2 + .../LocalStrings_en_US.properties | 2 + .../LocalStrings_ja_JP.properties | 2 + .../LocalStrings_zh_CN.properties | 2 + .../LocalStrings_zh_HK.properties | 2 + .../LocalStrings_zh_TW.properties | 2 + .../src/main/resources/plugin.json | 7 + .../main/resources/plugin_job_template.json | 15 + package.xml | 14 + pom.xml | 2 + 33 files changed, 2321 insertions(+) create mode 100644 cassandrareader/doc/cassandrareader.md create mode 100644 cassandrareader/pom.xml create mode 100644 cassandrareader/src/main/assembly/package.xml create mode 100644 cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReader.java create mode 100644 cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderErrorCode.java create mode 100644 cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderHelper.java create mode 100644 cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/Key.java create mode 100644 cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings.properties create mode 100644 cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_en_US.properties create mode 100644 cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_ja_JP.properties create mode 100644 cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_CN.properties create mode 100644 cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_HK.properties create mode 100644 cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_TW.properties create mode 100644 cassandrareader/src/main/resources/plugin.json create mode 100644 cassandrareader/src/main/resources/plugin_job_template.json create mode 100644 cassandrawriter/doc/cassandrawriter.md create mode 100644 cassandrawriter/pom.xml create mode 100644 cassandrawriter/src/main/assembly/package.xml create mode 100644 cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriter.java create mode 100644 cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterErrorCode.java create mode 100644 cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterHelper.java create mode 100644 cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/Key.java create mode 100644 cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings.properties create mode 100644 cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_en_US.properties create mode 100644 cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_ja_JP.properties create mode 100644 cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_CN.properties create mode 100644 cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_HK.properties create mode 100644 cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_TW.properties create mode 100644 cassandrawriter/src/main/resources/plugin.json create mode 100644 cassandrawriter/src/main/resources/plugin_job_template.json diff --git a/README.md b/README.md index dff02301..e806ba2b 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N | | Phoenix5.x | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/hbase20xsqlreader/doc/hbase20xsqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hbase20xsqlwriter/doc/hbase20xsqlwriter.md)| | | MongoDB | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/mongoreader/doc/mongoreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/mongowriter/doc/mongowriter.md)| | | Hive | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/hdfsreader/doc/hdfsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md)| +| | Cassandra | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/cassandrareader/doc/cassandrareader.md) 、[写](https://github.com/alibaba/DataX/blob/master/cassandrawriter/doc/cassandrawriter.md)| | 无结构化数据存储 | TxtFile | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/txtfilereader/doc/txtfilereader.md) 、[写](https://github.com/alibaba/DataX/blob/master/txtfilewriter/doc/txtfilewriter.md)| | | FTP | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/ftpreader/doc/ftpreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/ftpwriter/doc/ftpwriter.md)| | | HDFS | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/hdfsreader/doc/hdfsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md)| diff --git a/cassandrareader/doc/cassandrareader.md b/cassandrareader/doc/cassandrareader.md new file mode 100644 index 00000000..144e78ae --- /dev/null +++ b/cassandrareader/doc/cassandrareader.md @@ -0,0 +1,217 @@ + +# CassandraReader 插件文档 + + +___ + + + +## 1 快速介绍 + +CassandraReader插件实现了从Cassandra读取数据。在底层实现上，CassandraReader通过datastax的java driver连接Cassandra实例，并执行相应的cql语句将数据从cassandra中SELECT出来。 + + +## 2 实现原理 + +简而言之，CassandraReader通过java driver连接到Cassandra实例，并根据用户配置的信息生成查询SELECT CQL语句，然后发送到Cassandra，并将该CQL执行返回结果使用DataX自定义的数据类型拼装为抽象的数据集，并传递给下游Writer处理。 + +对于用户配置Table、Column的信息，CassandraReader将其拼接为CQL语句发送到Cassandra。 + + +## 3 功能说明 + +### 3.1 配置样例 + +* 配置一个从Cassandra同步抽取数据到本地的作业: + +``` +{ + "job": { + "setting": { + "speed": { + "channel": 3 + } + }, + "content": [ + { + "reader": { + "name": "cassandrareader", + "parameter": { + "host": "localhost", + "port": 9042, + "useSSL": false, + "keyspace": "test", + "table": "datax_src", + "column": [ + "textCol", + "blobCol", + "writetime(blobCol)", + "boolCol", + "smallintCol", + "tinyintCol", + "intCol", + "bigintCol", + "varintCol", + "floatCol", + "doubleCol", + "decimalCol", + "dateCol", + "timeCol", + "timeStampCol", + "uuidCol", + "inetCol", + "durationCol", + "listCol", + "mapCol", + "setCol" + "tupleCol" + "udtCol", + ] + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "print":true + } + } + } + ] + } +} + +``` + + +### 3.2 参数说明 + +* **host** + + * 描述：Cassandra连接点的域名或ip，多个node之间用逗号分隔。
+ + * 必选：是
+ + * 默认值：无
+ +* **port** + + * 描述：Cassandra端口。
+ + * 必选：是
+ + * 默认值：9042
+ +* **username** + + * 描述：数据源的用户名
+ + * 必选：否
+ + * 默认值：无
+ +* **password** + + * 描述：数据源指定用户名的密码
+ + * 必选：否
+ + * 默认值：无
+ +* **useSSL** + + * 描述：是否使用SSL连接。
+ + * 必选：否
+ + * 默认值：false
+ +* **keyspace** + + * 描述：需要同步的表所在的keyspace。
+ + * 必选：是
+ + * 默认值：无
+ +* **table** + + * 描述：所选取的需要同步的表。
+ + * 必选：是
+ + * 默认值：无
+ +* **column** + + * 描述：所配置的表中需要同步的列集合。
+ 其中的元素可以指定列的名称或writetime(column_name)，后一种形式会读取column_name列的时间戳而不是数据。 + + * 必选：是
+ + * 默认值：无
+ + +* **where** + + * 描述：数据筛选条件的cql表达式，例如:
+ ``` + "where":"textcol='a'" + ``` + + * 必选：否
+ + * 默认值：无
+ +* **allowFiltering** + + * 描述：是否在服务端过滤数据。参考cassandra文档中ALLOW FILTERING关键字的相关描述。
+ + * 必选：否
+ + * 默认值：无
+ +* **consistancyLevel** + + * 描述：数据一致性级别。可选ONE|QUORUM|LOCAL_QUORUM|EACH_QUORUM|ALL|ANY|TWO|THREE|LOCAL_ONE
+ + * 必选：否
+ + * 默认值：LOCAL_QUORUM
+ + +### 3.3 类型转换 + +目前CassandraReader支持除counter和Custom类型之外的所有类型。 + +下面列出CassandraReader针对Cassandra类型转换列表: + + +| DataX 内部类型| Cassandra 数据类型 | +| -------- | ----- | +| Long |int, tinyint, smallint,varint,bigint,time| +| Double |float, double, decimal| +| String |ascii,varchar, text,uuid,timeuuid,duration,list,map,set,tuple,udt,inet | +| Date |date, timestamp | +| Boolean |bool | +| Bytes |blob | + + + +请注意: + +* 目前不支持counter类型和custom类型。 + +## 4 性能报告 + +略 + +## 5 约束限制 + +### 5.1 主备同步数据恢复问题 + +略 + +## 6 FAQ + + + diff --git a/cassandrareader/pom.xml b/cassandrareader/pom.xml new file mode 100644 index 00000000..4acc5e3a --- /dev/null +++ b/cassandrareader/pom.xml @@ -0,0 +1,133 @@ + + + 4.0.0 + + com.alibaba.datax + datax-all + 0.0.1-SNAPSHOT + + cassandrareader + cassandrareader + jar + + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + + com.datastax.cassandra + cassandra-driver-core + 3.7.2 + shaded + + + com.google.guava + guava + + + + + com.google.guava + guava + 16.0.1 + + + commons-codec + commons-codec + 1.9 + + + + + junit + junit + test + + + com.alibaba.datax + datax-core + ${datax-project-version} + + + com.alibaba.datax + datax-service-face + + + org.apache.hadoop + hadoop-common + + + org.apache.hive + hive-exec + + + org.apache.hive + hive-serde + + + javolution + javolution + + + test + + + org.mockito + mockito-all + 1.9.5 + test + + + + + + + + + maven-compiler-plugin + + 1.6 + 1.6 + ${project-sourceEncoding} + + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + diff --git a/cassandrareader/src/main/assembly/package.xml b/cassandrareader/src/main/assembly/package.xml new file mode 100644 index 00000000..263aa345 --- /dev/null +++ b/cassandrareader/src/main/assembly/package.xml @@ -0,0 +1,35 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/reader/cassandrareader + + + target/ + + cassandrareader-0.0.1-SNAPSHOT.jar + + plugin/reader/cassandrareader + + + + + + false + plugin/reader/cassandrareader/libs + runtime + + + diff --git a/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReader.java b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReader.java new file mode 100644 index 00000000..e5215e64 --- /dev/null +++ b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReader.java @@ -0,0 +1,123 @@ +package com.alibaba.datax.plugin.reader.cassandrareader; + +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.common.spi.Reader; +import com.alibaba.datax.common.util.Configuration; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Row; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SimpleStatement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class CassandraReader extends Reader { + private static final Logger LOG = LoggerFactory + .getLogger(CassandraReader.class); + + public static class Job extends Reader.Job { + + private Configuration jobConfig = null; + private Cluster cluster = null; + + @Override public void init() { + this.jobConfig = super.getPluginJobConf(); + this.jobConfig = super.getPluginJobConf(); + String username = jobConfig.getString(Key.USERNAME); + String password = jobConfig.getString(Key.PASSWORD); + String hosts = jobConfig.getString(Key.HOST); + Integer port = jobConfig.getInt(Key.PORT,9042); + boolean useSSL = jobConfig.getBool(Key.USESSL); + + if ((username != null) && !username.isEmpty()) { + Cluster.Builder clusterBuilder = Cluster.builder().withCredentials(username, password) + .withPort(Integer.valueOf(port)).addContactPoints(hosts.split(",")); + if (useSSL) { + clusterBuilder = clusterBuilder.withSSL(); + } + cluster = clusterBuilder.build(); + } else { + cluster = Cluster.builder().withPort(Integer.valueOf(port)) + .addContactPoints(hosts.split(",")).build(); + } + CassandraReaderHelper.checkConfig(jobConfig,cluster); + } + + @Override public void destroy() { + + } + + @Override public List split(int adviceNumber) { + List splittedConfigs = CassandraReaderHelper.splitJob(adviceNumber,jobConfig,cluster); + return splittedConfigs; + } + + } + + public static class Task extends Reader.Task { + private Configuration taskConfig; + private Cluster cluster = null; + private Session session = null; + private String queryString = null; + private ConsistencyLevel consistencyLevel; + private int columnNumber = 0; + private List columnMeta = null; + + @Override public void init() { + this.taskConfig = super.getPluginJobConf(); + String username = taskConfig.getString(Key.USERNAME); + String password = taskConfig.getString(Key.PASSWORD); + String hosts = taskConfig.getString(Key.HOST); + Integer port = taskConfig.getInt(Key.PORT); + boolean useSSL = taskConfig.getBool(Key.USESSL); + String keyspace = taskConfig.getString(Key.KEYSPACE); + this.columnMeta = taskConfig.getList(Key.COLUMN,String.class); + columnNumber = columnMeta.size(); + + if ((username != null) && !username.isEmpty()) { + Cluster.Builder clusterBuilder = Cluster.builder().withCredentials(username, password) + .withPort(Integer.valueOf(port)).addContactPoints(hosts.split(",")); + if (useSSL) { + clusterBuilder = clusterBuilder.withSSL(); + } + cluster = clusterBuilder.build(); + } else { + cluster = Cluster.builder().withPort(Integer.valueOf(port)) + .addContactPoints(hosts.split(",")).build(); + } + session = cluster.connect(keyspace); + String cl = taskConfig.getString(Key.CONSITANCY_LEVEL); + if( cl != null && !cl.isEmpty() ) { + consistencyLevel = ConsistencyLevel.valueOf(cl); + } else { + consistencyLevel = ConsistencyLevel.LOCAL_QUORUM; + } + + queryString = CassandraReaderHelper.getQueryString(taskConfig,cluster); + LOG.info("query = " + queryString); + + } + + @Override public void startRead(RecordSender recordSender) { + ResultSet r = session.execute(new SimpleStatement(queryString).setConsistencyLevel(consistencyLevel)); + for (Row row : r ) { + Record record = recordSender.createRecord(); + record = CassandraReaderHelper.buildRecord(record,row,r.getColumnDefinitions(),columnNumber, + super.getTaskPluginCollector()); + if( record != null ) + recordSender.sendToWriter(record); + } + } + + @Override public void destroy() { + + } + + } + +} diff --git a/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderErrorCode.java b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderErrorCode.java new file mode 100644 index 00000000..42927ef8 --- /dev/null +++ b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderErrorCode.java @@ -0,0 +1,32 @@ +package com.alibaba.datax.plugin.reader.cassandrareader; + +import com.alibaba.datax.common.spi.ErrorCode; + +public enum CassandraReaderErrorCode implements ErrorCode { + CONF_ERROR("CassandraReader-00", "配置错误."), + ; + + private final String code; + private final String description; + + private CassandraReaderErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s]. ", this.code, + this.description); + } +} diff --git a/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderHelper.java b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderHelper.java new file mode 100644 index 00000000..0a4e83fa --- /dev/null +++ b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderHelper.java @@ -0,0 +1,607 @@ +package com.alibaba.datax.plugin.reader.cassandrareader; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.alibaba.datax.common.element.BoolColumn; +import com.alibaba.datax.common.element.BytesColumn; +import com.alibaba.datax.common.element.DateColumn; +import com.alibaba.datax.common.element.DoubleColumn; +import com.alibaba.datax.common.element.LongColumn; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.element.StringColumn; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.fastjson.JSON; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.CodecRegistry; +import com.datastax.driver.core.ColumnDefinitions; +import com.datastax.driver.core.ColumnMetadata; +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.Duration; +import com.datastax.driver.core.LocalDate; +import com.datastax.driver.core.Row; +import com.datastax.driver.core.TableMetadata; +import com.datastax.driver.core.TupleType; +import com.datastax.driver.core.TupleValue; +import com.datastax.driver.core.UDTValue; +import com.datastax.driver.core.UserType; +import com.google.common.reflect.TypeToken; +import org.apache.commons.codec.binary.Base64; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Created by mazhenlin on 2019/8/21. + */ +public class CassandraReaderHelper { + static CodecRegistry registry = new CodecRegistry(); + private static final Logger LOG = LoggerFactory + .getLogger(CassandraReader.class); + + static class TypeNotSupported extends Exception{} + + static String toJSonString(Object o, DataType type ) throws Exception{ + if( o == null ) return JSON.toJSONString(null); + switch (type.getName()) { + case LIST: + case MAP: + case SET: + case TUPLE: + case UDT: + return JSON.toJSONString(transferObjectForJson(o,type)); + + default: + return JSON.toJSONString(o); + } + } + + static Object transferObjectForJson(Object o,DataType type) throws TypeNotSupported{ + if( o == null ) return o; + switch (type.getName()) { + case ASCII: + case TEXT: + case VARCHAR: + case BOOLEAN: + case SMALLINT: + case TINYINT: + case INT: + case BIGINT: + case VARINT: + case FLOAT: + case DOUBLE: + case DECIMAL: + case UUID: + case TIMEUUID: + case TIME: + return o; + + case BLOB: + ByteBuffer byteBuffer = (ByteBuffer)o; + String s = Base64.encodeBase64String( + Arrays.copyOfRange(byteBuffer.array(),byteBuffer.position(), + byteBuffer.limit())); + return s; + + case DATE: + return ((LocalDate)o).getMillisSinceEpoch(); + + case TIMESTAMP: + return ((Date)o).getTime(); + + case DURATION: + return o.toString(); + + case INET: + return ((InetAddress)o).getHostAddress(); + + case LIST: { + return transferListForJson((List)o,type.getTypeArguments().get(0)); + } + + case MAP: { + DataType keyType = type.getTypeArguments().get(0); + DataType valType = type.getTypeArguments().get(1); + return transferMapForJson((Map)o,keyType,valType); + } + + case SET: { + return transferSetForJson((Set)o, type.getTypeArguments().get(0)); + } + + case TUPLE: { + return transferTupleForJson((TupleValue)o,((TupleType)type).getComponentTypes()); + } + + case UDT: { + return transferUDTForJson((UDTValue)o); + } + + default: + throw new TypeNotSupported(); + } + + } + + static List transferListForJson(List clist, DataType eleType) throws TypeNotSupported { + List result = new ArrayList(); + switch (eleType.getName()) { + case ASCII: + case TEXT: + case VARCHAR: + case BOOLEAN: + case SMALLINT: + case TINYINT: + case INT: + case BIGINT: + case VARINT: + case FLOAT: + case DOUBLE: + case DECIMAL: + case TIME: + case UUID: + case TIMEUUID: + return clist; + + case BLOB: + case DATE: + case TIMESTAMP: + case DURATION: + case INET: + case LIST: + case MAP: + case SET: + case TUPLE: + case UDT: + for (Object item : clist) { + Object newItem = transferObjectForJson(item, eleType); + result.add(newItem); + } + break; + + default: + throw new TypeNotSupported(); + } + + return result; + } + + static Set transferSetForJson(Set cset,DataType eleType) throws TypeNotSupported{ + Set result = new HashSet(); + switch (eleType.getName()) { + case ASCII: + case TEXT: + case VARCHAR: + case BOOLEAN: + case SMALLINT: + case TINYINT: + case INT: + case BIGINT: + case VARINT: + case FLOAT: + case DOUBLE: + case DECIMAL: + case TIME: + case UUID: + case TIMEUUID: + return cset; + + case BLOB: + case DATE: + case TIMESTAMP: + case DURATION: + case INET: + case LIST: + case MAP: + case SET: + case TUPLE: + case UDT: + for (Object item : cset) { + Object newItem = transferObjectForJson(item,eleType); + result.add(newItem); + } + break; + + default: + throw new TypeNotSupported(); + } + + return result; + } + + static Map transferMapForJson(Map cmap,DataType keyType,DataType valueType) throws TypeNotSupported { + Map newMap = new HashMap(); + for( Object e : cmap.entrySet() ) { + Object k = ((Map.Entry)e).getKey(); + Object v = ((Map.Entry)e).getValue(); + Object newKey = transferObjectForJson(k,keyType); + Object newValue = transferObjectForJson(v,valueType); + if( !(newKey instanceof String) ) { + newKey = JSON.toJSONString(newKey); + } + newMap.put(newKey,newValue); + } + return newMap; + } + + static List transferTupleForJson(TupleValue tupleValue,List componentTypes) throws TypeNotSupported { + List l = new ArrayList(); + for (int j = 0; j < componentTypes.size(); j++ ) { + DataType dataType = componentTypes.get(j); + TypeToken eltClass = registry.codecFor(dataType).getJavaType(); + Object ele = tupleValue.get(j,eltClass); + l.add(transferObjectForJson(ele,dataType)); + } + return l; + } + + static Map transferUDTForJson(UDTValue udtValue) throws TypeNotSupported { + Map newMap = new HashMap(); + int j = 0; + for (UserType.Field f : udtValue.getType()) { + DataType dataType = f.getType(); + TypeToken eltClass = registry.codecFor(dataType).getJavaType(); + Object ele = udtValue.get(j, eltClass); + newMap.put(f.getName(),transferObjectForJson(ele,dataType)); + j++; + } + return newMap; + } + + static Record buildRecord(Record record, Row rs, ColumnDefinitions metaData, int columnNumber, + TaskPluginCollector taskPluginCollector) { + + try { + for (int i = 0; i < columnNumber; i++) + try { + if (rs.isNull(i)) { + record.addColumn(new StringColumn()); + continue; + } + switch (metaData.getType(i).getName()) { + + case ASCII: + case TEXT: + case VARCHAR: + record.addColumn(new StringColumn(rs.getString(i))); + break; + + case BLOB: + record.addColumn(new BytesColumn(rs.getBytes(i).array())); + break; + + case BOOLEAN: + record.addColumn(new BoolColumn(rs.getBool(i))); + break; + + case SMALLINT: + record.addColumn(new LongColumn((int)rs.getShort(i))); + break; + + case TINYINT: + record.addColumn(new LongColumn((int)rs.getByte(i))); + break; + + case INT: + record.addColumn(new LongColumn(rs.getInt(i))); + break; + + case BIGINT: + record.addColumn(new LongColumn(rs.getLong(i))); + break; + + case VARINT: + record.addColumn(new LongColumn(rs.getVarint(i))); + break; + + case FLOAT: + record.addColumn(new DoubleColumn(rs.getFloat(i))); + break; + + case DOUBLE: + record.addColumn(new DoubleColumn(rs.getDouble(i))); + break; + + case DECIMAL: + record.addColumn(new DoubleColumn(rs.getDecimal(i))); + break; + + case DATE: + record.addColumn(new DateColumn(rs.getDate(i).getMillisSinceEpoch())); + break; + + case TIME: + record.addColumn(new LongColumn(rs.getTime(i))); + break; + + case TIMESTAMP: + record.addColumn(new DateColumn(rs.getTimestamp(i))); + break; + + case UUID: + case TIMEUUID: + record.addColumn(new StringColumn(rs.getUUID(i).toString())); + break; + + case INET: + record.addColumn(new StringColumn(rs.getInet(i).getHostAddress())); + break; + + case DURATION: + record.addColumn(new StringColumn(rs.get(i,Duration.class).toString())); + break; + + case LIST: { + TypeToken listEltClass = registry.codecFor(metaData.getType(i).getTypeArguments().get(0)).getJavaType(); + List l = rs.getList(i, listEltClass); + record.addColumn(new StringColumn(toJSonString(l,metaData.getType(i)))); + } + break; + + case MAP: { + DataType keyType = metaData.getType(i).getTypeArguments().get(0); + DataType valType = metaData.getType(i).getTypeArguments().get(1); + TypeToken keyEltClass = registry.codecFor(keyType).getJavaType(); + TypeToken valEltClass = registry.codecFor(valType).getJavaType(); + Map m = rs.getMap(i, keyEltClass, valEltClass); + record.addColumn(new StringColumn(toJSonString(m,metaData.getType(i)))); + } + break; + + case SET: { + TypeToken setEltClass = registry.codecFor(metaData.getType(i).getTypeArguments().get(0)) + .getJavaType(); + Set set = rs.getSet(i, setEltClass); + record.addColumn(new StringColumn(toJSonString(set,metaData.getType(i)))); + } + break; + + case TUPLE: { + TupleValue t = rs.getTupleValue(i); + record.addColumn(new StringColumn(toJSonString(t,metaData.getType(i)))); + } + break; + + case UDT: { + UDTValue t = rs.getUDTValue(i); + record.addColumn(new StringColumn(toJSonString(t,metaData.getType(i)))); + } + break; + + default: + throw DataXException + .asDataXException( + CassandraReaderErrorCode.CONF_ERROR, + String.format( + "您的配置文件中的列配置信息有误. 因为DataX 不支持数据库读取这种字段类型. 字段名:[%s], " + + "字段类型:[%s]. ", + metaData.getName(i), + metaData.getType(i))); + } + } catch (TypeNotSupported t) { + throw DataXException + .asDataXException( + CassandraReaderErrorCode.CONF_ERROR, + String.format( + "您的配置文件中的列配置信息有误. 因为DataX 不支持数据库读取这种字段类型. 字段名:[%s], " + + "字段类型:[%s]. ", + metaData.getName(i), + metaData.getType(i))); + + } + } catch (Exception e) { + //TODO 这里识别为脏数据靠谱吗？ + taskPluginCollector.collectDirtyRecord(record, e); + if (e instanceof DataXException) { + throw (DataXException) e; + } + return null; + } + return record; + } + + public static List splitJob(int adviceNumber,Configuration jobConfig,Cluster cluster) { + List splittedConfigs = new ArrayList(); + if( adviceNumber <= 1 ) { + splittedConfigs.add(jobConfig); + return splittedConfigs; + } + String where = jobConfig.getString(Key.WHERE); + if(where != null && where.toLowerCase().contains("token(")) { + splittedConfigs.add(jobConfig); + return splittedConfigs; + } + String partitioner = cluster.getMetadata().getPartitioner(); + if( partitioner.endsWith("RandomPartitioner")) { + BigDecimal minToken = BigDecimal.valueOf(-1); + BigDecimal maxToken = new BigDecimal(new BigInteger("2").pow(127)); + BigDecimal step = maxToken.subtract(minToken) + .divide(BigDecimal.valueOf(adviceNumber),2, BigDecimal.ROUND_HALF_EVEN); + for ( int i = 0; i < adviceNumber; i++ ) { + BigInteger l = minToken.add(step.multiply(BigDecimal.valueOf(i))).toBigInteger(); + BigInteger r = minToken.add(step.multiply(BigDecimal.valueOf(i+1))).toBigInteger(); + if( i == adviceNumber - 1 ) { + r = maxToken.toBigInteger(); + } + Configuration taskConfig = jobConfig.clone(); + taskConfig.set(Key.MIN_TOKEN,l.toString()); + taskConfig.set(Key.MAX_TOKEN,r.toString()); + splittedConfigs.add(taskConfig); + } + } + else if( partitioner.endsWith("Murmur3Partitioner") ) { + BigDecimal minToken = BigDecimal.valueOf(Long.MIN_VALUE); + BigDecimal maxToken = BigDecimal.valueOf(Long.MAX_VALUE); + BigDecimal step = maxToken.subtract(minToken) + .divide(BigDecimal.valueOf(adviceNumber),2, BigDecimal.ROUND_HALF_EVEN); + for ( int i = 0; i < adviceNumber; i++ ) { + long l = minToken.add(step.multiply(BigDecimal.valueOf(i))).longValue(); + long r = minToken.add(step.multiply(BigDecimal.valueOf(i+1))).longValue(); + if( i == adviceNumber - 1 ) { + r = maxToken.longValue(); + } + Configuration taskConfig = jobConfig.clone(); + taskConfig.set(Key.MIN_TOKEN,String.valueOf(l)); + taskConfig.set(Key.MAX_TOKEN,String.valueOf(r)); + splittedConfigs.add(taskConfig); + } + } + else { + splittedConfigs.add(jobConfig); + } + return splittedConfigs; + } + + public static String getQueryString(Configuration taskConfig,Cluster cluster) { + List columnMeta = taskConfig.getList(Key.COLUMN,String.class); + String keyspace = taskConfig.getString(Key.KEYSPACE); + String table = taskConfig.getString(Key.TABLE); + + StringBuilder columns = new StringBuilder(); + for( String column : columnMeta ) { + if(columns.length() > 0 ) { + columns.append(","); + } + columns.append(column); + } + + StringBuilder where = new StringBuilder(); + String whereString = taskConfig.getString(Key.WHERE); + if( whereString != null && !whereString.isEmpty() ) { + where.append(whereString); + } + String minToken = taskConfig.getString(Key.MIN_TOKEN); + String maxToken = taskConfig.getString(Key.MAX_TOKEN); + if( minToken !=null || maxToken !=null ) { + LOG.info("range:" + minToken + "~" + maxToken); + List pks = cluster.getMetadata().getKeyspace(keyspace).getTable(table).getPartitionKey(); + StringBuilder sb = new StringBuilder(); + for( ColumnMetadata pk : pks ) { + if( sb.length() > 0 ) { + sb.append(","); + } + sb.append(pk.getName()); + } + String s = sb.toString(); + if (minToken != null && !minToken.isEmpty()) { + if( where.length() > 0 ){ + where.append(" AND "); + } + where.append("token(").append(s).append(")").append(" > ").append(minToken); + } + if (maxToken != null && !maxToken.isEmpty()) { + if( where.length() > 0 ){ + where.append(" AND "); + } + where.append("token(").append(s).append(")").append(" <= ").append(maxToken); + } + } + + boolean allowFiltering = taskConfig.getBool(Key.ALLOW_FILTERING,false); + + StringBuilder select = new StringBuilder(); + select.append("SELECT ").append(columns.toString()).append(" FROM ").append(table); + if( where.length() > 0 ){ + select.append(" where ").append(where.toString()); + } + if( allowFiltering ) { + select.append(" ALLOW FILTERING"); + } + select.append(";"); + return select.toString(); + } + + public static void checkConfig(Configuration jobConfig,Cluster cluster) { + ensureStringExists(jobConfig,Key.HOST); + ensureStringExists(jobConfig,Key.KEYSPACE); + ensureStringExists(jobConfig,Key.TABLE); + ensureExists(jobConfig,Key.COLUMN); + + ///keyspace,table是否存在 + String keyspace = jobConfig.getString(Key.KEYSPACE); + if( cluster.getMetadata().getKeyspace(keyspace) == null ) { + throw DataXException + .asDataXException( + CassandraReaderErrorCode.CONF_ERROR, + String.format( + "配置信息有错误.keyspace'%s'不存在 .", + keyspace)); + } + String table = jobConfig.getString(Key.TABLE); + TableMetadata tableMetadata = cluster.getMetadata().getKeyspace(keyspace).getTable(table); + if( tableMetadata == null ) { + throw DataXException + .asDataXException( + CassandraReaderErrorCode.CONF_ERROR, + String.format( + "配置信息有错误.表'%s'不存在 .", + table)); + } + List columns = jobConfig.getList(Key.COLUMN,String.class); + for( String name : columns ) { + if( name == null || name.isEmpty() ) { + throw DataXException + .asDataXException( + CassandraReaderErrorCode.CONF_ERROR, + String.format( + "配置信息有错误.列信息中需要包含'%s'字段 .",Key.COLUMN_NAME)); + } + if( name.startsWith(Key.WRITE_TIME) ) { + String colName = name.substring(Key.WRITE_TIME.length(),name.length() - 1 ); + ColumnMetadata col = tableMetadata.getColumn(colName); + if( col == null ) { + throw DataXException + .asDataXException( + CassandraReaderErrorCode.CONF_ERROR, + String.format( + "配置信息有错误.列'%s'不存在 .",colName)); + } + } else { + ColumnMetadata col = tableMetadata.getColumn(name); + if( col == null ) { + throw DataXException + .asDataXException( + CassandraReaderErrorCode.CONF_ERROR, + String.format( + "配置信息有错误.列'%s'不存在 .",name)); + } + } + } + } + + static void ensureExists(Configuration jobConfig,String keyword) { + if( jobConfig.get(keyword) == null ) { + throw DataXException + .asDataXException( + CassandraReaderErrorCode.CONF_ERROR, + String.format( + "配置信息有错误.参数'%s'为必填项 .", + keyword)); + } + } + + static void ensureStringExists(Configuration jobConfig,String keyword) { + ensureExists(jobConfig,keyword); + if( jobConfig.getString(keyword).isEmpty() ) { + throw DataXException + .asDataXException( + CassandraReaderErrorCode.CONF_ERROR, + String.format( + "配置信息有错误.参数'%s'不能为空 .", + keyword)); + } + } + +} diff --git a/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/Key.java b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/Key.java new file mode 100644 index 00000000..73a9fadf --- /dev/null +++ b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/Key.java @@ -0,0 +1,39 @@ +package com.alibaba.datax.plugin.reader.cassandrareader; + +/** + * Created by mazhenlin on 2019/8/19. + */ +public class Key { + public final static String USERNAME = "username"; + public final static String PASSWORD = "password"; + + public final static String HOST = "host"; + public final static String PORT = "port"; + public final static String USESSL = "useSSL"; + + public final static String KEYSPACE = "keyspace"; + public final static String TABLE = "table"; + public final static String COLUMN = "column"; + public final static String WHERE = "where"; + public final static String ALLOW_FILTERING = "allowFiltering"; + public final static String CONSITANCY_LEVEL = "consistancyLevel"; + public final static String MIN_TOKEN = "minToken"; + public final static String MAX_TOKEN = "maxToken"; + + /** + * 每个列的名字 + */ + public static final String COLUMN_NAME = "name"; + /** + * 列分隔符 + */ + public static final String COLUMN_SPLITTER = "format"; + public static final String WRITE_TIME = "writetime("; + public static final String ELEMENT_SPLITTER = "splitter"; + public static final String ENTRY_SPLITTER = "entrySplitter"; + public static final String KV_SPLITTER = "kvSplitter"; + public static final String ELEMENT_CONFIG = "element"; + public static final String TUPLE_CONNECTOR = "_"; + public static final String KEY_CONFIG = "key"; + public static final String VALUE_CONFIG = "value"; +} diff --git a/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings.properties b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings.properties new file mode 100644 index 00000000..7f4cb92d --- /dev/null +++ b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings.properties @@ -0,0 +1 @@ +errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF \ No newline at end of file diff --git a/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_en_US.properties b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_en_US.properties new file mode 100644 index 00000000..e69de29b diff --git a/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_ja_JP.properties b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_ja_JP.properties new file mode 100644 index 00000000..7f4cb92d --- /dev/null +++ b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_ja_JP.properties @@ -0,0 +1 @@ +errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF \ No newline at end of file diff --git a/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_CN.properties b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_CN.properties new file mode 100644 index 00000000..7f4cb92d --- /dev/null +++ b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_CN.properties @@ -0,0 +1 @@ +errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF \ No newline at end of file diff --git a/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_HK.properties b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_HK.properties new file mode 100644 index 00000000..7f4cb92d --- /dev/null +++ b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_HK.properties @@ -0,0 +1 @@ +errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF \ No newline at end of file diff --git a/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_TW.properties b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_TW.properties new file mode 100644 index 00000000..7f4cb92d --- /dev/null +++ b/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_TW.properties @@ -0,0 +1 @@ +errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF \ No newline at end of file diff --git a/cassandrareader/src/main/resources/plugin.json b/cassandrareader/src/main/resources/plugin.json new file mode 100644 index 00000000..fec21db7 --- /dev/null +++ b/cassandrareader/src/main/resources/plugin.json @@ -0,0 +1,6 @@ +{ + "name": "cassandrareader", + "class": "com.alibaba.datax.plugin.reader.cassandrareader.CassandraReader", + "description": "useScene: prod. mechanism: execute select cql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", + "developer": "alibaba" +} \ No newline at end of file diff --git a/cassandrareader/src/main/resources/plugin_job_template.json b/cassandrareader/src/main/resources/plugin_job_template.json new file mode 100644 index 00000000..a49176a2 --- /dev/null +++ b/cassandrareader/src/main/resources/plugin_job_template.json @@ -0,0 +1,15 @@ +{ + "name": "cassandrareader", + "parameter": { + "username": "", + "password": "", + "host": "", + "port": "", + "useSSL": false, + "keyspace": "", + "table": "", + "column": [ + "c1","c2","c3" + ] + } +} \ No newline at end of file diff --git a/cassandrawriter/doc/cassandrawriter.md b/cassandrawriter/doc/cassandrawriter.md new file mode 100644 index 00000000..ca99d2a0 --- /dev/null +++ b/cassandrawriter/doc/cassandrawriter.md @@ -0,0 +1,227 @@ + +# CassandraWriter 插件文档 + + +___ + + + +## 1 快速介绍 + +CassandraWriter插件实现了向Cassandra写入数据。在底层实现上，CassandraWriter通过datastax的java driver连接Cassandra实例，并执行相应的cql语句将数据写入cassandra中。 + + +## 2 实现原理 + +简而言之，CassandraWriter通过java driver连接到Cassandra实例，并根据用户配置的信息生成INSERT CQL语句，然后发送到Cassandra。 + +对于用户配置Table、Column的信息，CassandraReader将其拼接为CQL语句发送到Cassandra。 + + +## 3 功能说明 + +### 3.1 配置样例 + +* 配置一个从内存产生到Cassandra导入的作业: + +``` +{ + "job": { + "setting": { + "speed": { + "channel": 5 + } + }, + "content": [ + { + "reader": { + "name": "streamreader", + "parameter": { + "column": [ + {"value":"name","type": "string"}, + {"value":"false","type":"bool"}, + {"value":"1988-08-08 08:08:08","type":"date"}, + {"value":"addr","type":"bytes"}, + {"value":1.234,"type":"double"}, + {"value":12345678,"type":"long"}, + {"value":2.345,"type":"double"}, + {"value":3456789,"type":"long"}, + {"value":"4a0ef8c0-4d97-11d0-db82-ebecdb03ffa5","type":"string"}, + {"value":"value","type":"bytes"}, + {"value":"-838383838,37377373,-383883838,27272772,393993939,-38383883,83883838,-1350403181,817650816,1630642337,251398784,-622020148","type":"string"}, + ], + "sliceRecordCount": 10000000 + } + }, + "writer": { + "name": "cassandrawriter", + "parameter": { + "host": "localhost", + "port": 9042, + "useSSL": false, + "keyspace": "stresscql", + "table": "dst", + "batchSize":10, + "column": [ + "name", + "choice", + "date", + "address", + "dbl", + "lval", + "fval", + "ival", + "uid", + "value", + "listval" + ] + } + } + } + ] + } +} +``` + + +### 3.2 参数说明 + +* **host** + + * 描述：Cassandra连接点的域名或ip，多个node之间用逗号分隔。
+ + * 必选：是
+ + * 默认值：无
+ +* **port** + + * 描述：Cassandra端口。
+ + * 必选：是
+ + * 默认值：9042
+ +* **username** + + * 描述：数据源的用户名
+ + * 必选：否
+ + * 默认值：无
+ +* **password** + + * 描述：数据源指定用户名的密码
+ + * 必选：否
+ + * 默认值：无
+ +* **useSSL** + + * 描述：是否使用SSL连接。
+ + * 必选：否
+ + * 默认值：false
+ +* **connectionsPerHost** + + * 描述：客户端连接池配置：与服务器每个节点建多少个连接。
+ + * 必选：否
+ + * 默认值：8
+ +* **maxPendingPerConnection** + + * 描述：客户端连接池配置：每个连接最大请求数。
+ + * 必选：否
+ + * 默认值：128
+ +* **keyspace** + + * 描述：需要同步的表所在的keyspace。
+ + * 必选：是
+ + * 默认值：无
+ +* **table** + + * 描述：所选取的需要同步的表。
+ + * 必选：是
+ + * 默认值：无
+ +* **column** + + * 描述：所配置的表中需要同步的列集合。
+ 内容可以是列的名称或"writetime()"。如果将列名配置为writetime()，会将这一列的内容作为时间戳。 + + * 必选：是
+ + * 默认值：无
+ + +* **consistancyLevel** + + * 描述：数据一致性级别。可选ONE|QUORUM|LOCAL_QUORUM|EACH_QUORUM|ALL|ANY|TWO|THREE|LOCAL_ONE
+ + * 必选：否
+ + * 默认值：LOCAL_QUORUM
+ +* **batchSize** + + * 描述：一次批量提交(UNLOGGED BATCH)的记录数大小（条数）。注意batch的大小有如下限制：
+ （1）不能超过65535。
+ (2) batch中的内容大小受到服务器端batch_size_fail_threshold_in_kb的限制。
+ (3) 如果batch中的内容超过了batch_size_warn_threshold_in_kb的限制，会打出warn日志，但并不影响写入，忽略即可。
+ 如果批量提交失败，会把这个批量的所有内容重新逐条写入一遍。 + + * 必选：否
+ + * 默认值：1
+ + +### 3.3 类型转换 + +目前CassandraReader支持除counter和Custom类型之外的所有类型。 + +下面列出CassandraReader针对Cassandra类型转换列表: + + +| DataX 内部类型| Cassandra 数据类型 | +| -------- | ----- | +| Long |int, tinyint, smallint,varint,bigint,time| +| Double |float, double, decimal| +| String |ascii,varchar, text,uuid,timeuuid,duration,list,map,set,tuple,udt,inet | +| Date |date, timestamp | +| Boolean |bool | +| Bytes |blob | + + + +请注意: + +* 目前不支持counter类型和custom类型。 + +## 4 性能报告 + +略 + +## 5 约束限制 + +### 5.1 主备同步数据恢复问题 + +略 + +## 6 FAQ + + + diff --git a/cassandrawriter/pom.xml b/cassandrawriter/pom.xml new file mode 100644 index 00000000..01dfa62f --- /dev/null +++ b/cassandrawriter/pom.xml @@ -0,0 +1,125 @@ + + + + datax-all + com.alibaba.datax + 0.0.1-SNAPSHOT + + 4.0.0 + + cassandrawriter + cassandrawriter + 0.0.1-SNAPSHOT + jar + + + + + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + + + com.datastax.cassandra + cassandra-driver-core + 3.7.2 + + + commons-codec + commons-codec + 1.9 + + + + + junit + junit + test + + + com.alibaba.datax + datax-core + ${datax-project-version} + + + com.alibaba.datax + datax-service-face + + + org.apache.hadoop + hadoop-common + + + org.apache.hive + hive-exec + + + org.apache.hive + hive-serde + + + javolution + javolution + + + test + + + org.mockito + mockito-all + 1.9.5 + test + + + + + + + src/main/java + + **/*.properties + + + + + + + maven-compiler-plugin + + 1.6 + 1.6 + ${project-sourceEncoding} + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + + \ No newline at end of file diff --git a/cassandrawriter/src/main/assembly/package.xml b/cassandrawriter/src/main/assembly/package.xml new file mode 100644 index 00000000..b5706ab9 --- /dev/null +++ b/cassandrawriter/src/main/assembly/package.xml @@ -0,0 +1,35 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/writer/cassandrawriter + + + target/ + + cassandrawriter-0.0.1-SNAPSHOT.jar + + plugin/writer/cassandrawriter + + + + + + false + plugin/writer/cassandrawriter/libs + runtime + + + diff --git a/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriter.java b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriter.java new file mode 100644 index 00000000..3427ed15 --- /dev/null +++ b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriter.java @@ -0,0 +1,242 @@ +package com.alibaba.datax.plugin.writer.cassandrawriter; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.spi.Writer; +import com.alibaba.datax.common.util.Configuration; + +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.BatchStatement.Type; +import com.datastax.driver.core.BoundStatement; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.ColumnMetadata; +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.HostDistance; +import com.datastax.driver.core.PoolingOptions; +import com.datastax.driver.core.PreparedStatement; +import com.datastax.driver.core.ResultSetFuture; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.TableMetadata; +import com.datastax.driver.core.querybuilder.Insert; +import com.datastax.driver.core.querybuilder.QueryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.datastax.driver.core.querybuilder.QueryBuilder.timestamp; + +/** + * Created by mazhenlin on 2019/8/19. + */ +public class CassandraWriter extends Writer { + private static final Logger LOG = LoggerFactory + .getLogger(CassandraWriter.class); + public static class Job extends Writer.Job { + private Configuration originalConfig = null; + + @Override public List split(int mandatoryNumber) { + List splitResultConfigs = new ArrayList(); + for (int j = 0; j < mandatoryNumber; j++) { + splitResultConfigs.add(originalConfig.clone()); + } + return splitResultConfigs; + + } + + @Override public void init() { + originalConfig = getPluginJobConf(); + } + + @Override public void destroy() { + + } + } + + public static class Task extends Writer.Task { + private Configuration taskConfig; + private Cluster cluster = null; + private Session session = null; + private PreparedStatement statement = null; + private int columnNumber = 0; + private List columnTypes; + private List columnMeta = null; + private int writeTimeCol = -1; + private boolean asyncWrite = false; + private long batchSize = 1; + private List unConfirmedWrite; + private List bufferedWrite; + + @Override public void startWrite(RecordReceiver lineReceiver) { + try { + Record record; + while ((record = lineReceiver.getFromReader()) != null) { + if (record.getColumnNumber() != columnNumber) { + // 源头读取字段列数与目的表字段写入列数不相等，直接报错 + throw DataXException + .asDataXException( + CassandraWriterErrorCode.CONF_ERROR, + String.format( + "列配置信息有错误. 因为您配置的任务中，源头读取字段数:%s 与 目的表要写入的字段数:%s 不相等. 请检查您的配置并作出修改.", + record.getColumnNumber(), + this.columnNumber)); + } + + BoundStatement boundStmt = statement.bind(); + for (int i = 0; i < columnNumber; i++) { + if( writeTimeCol != -1 && i == writeTimeCol ) { + continue; + } + Column col = record.getColumn(i); + int pos = i; + if( writeTimeCol != -1 && pos > writeTimeCol ) { + pos = i - 1; + } + CassandraWriterHelper.setupColumn(boundStmt,pos,columnTypes.get(pos),col); + } + if(writeTimeCol != -1) { + Column col = record.getColumn(writeTimeCol ); + boundStmt.setLong(columnNumber - 1,col.asLong()); + } + if( batchSize <= 1 ) { + session.execute(boundStmt); + } else { + if( asyncWrite ) { + unConfirmedWrite.add(session.executeAsync(boundStmt)); + if (unConfirmedWrite.size() >= batchSize) { + for (ResultSetFuture write : unConfirmedWrite) { + write.getUninterruptibly(10000, TimeUnit.MILLISECONDS); + } + unConfirmedWrite.clear(); + } + } else { + bufferedWrite.add(boundStmt); + if( bufferedWrite.size() >= batchSize ) { + BatchStatement batchStatement = new BatchStatement(Type.UNLOGGED); + batchStatement.addAll(bufferedWrite); + try { + session.execute(batchStatement); + } catch (Exception e ) { + LOG.error("batch写入失败，尝试逐条写入.",e); + for( BoundStatement stmt: bufferedWrite ) { + session.execute(stmt); + } + } + ///LOG.info("batch finished. size = " + bufferedWrite.size()); + bufferedWrite.clear(); + } + } + } + + } + if( unConfirmedWrite != null && unConfirmedWrite.size() > 0 ) { + for( ResultSetFuture write : unConfirmedWrite ) { + write.getUninterruptibly(10000, TimeUnit.MILLISECONDS); + } + unConfirmedWrite.clear(); + } + if( bufferedWrite !=null && bufferedWrite.size() > 0 ) { + BatchStatement batchStatement = new BatchStatement(Type.UNLOGGED); + batchStatement.addAll(bufferedWrite); + session.execute(batchStatement); + bufferedWrite.clear(); + } + } catch (Exception e) { + throw DataXException.asDataXException( + CassandraWriterErrorCode.WRITE_DATA_ERROR, e); + } + } + + + @Override public void init() { + this.taskConfig = super.getPluginJobConf(); + String username = taskConfig.getString(Key.USERNAME); + String password = taskConfig.getString(Key.PASSWORD); + String hosts = taskConfig.getString(Key.HOST); + Integer port = taskConfig.getInt(Key.PORT,9042); + boolean useSSL = taskConfig.getBool(Key.USESSL); + String keyspace = taskConfig.getString(Key.KEYSPACE); + String table = taskConfig.getString(Key.TABLE); + batchSize = taskConfig.getLong(Key.BATCH_SIZE,1); + this.columnMeta = taskConfig.getList(Key.COLUMN,String.class); + columnTypes = new ArrayList(columnMeta.size()); + columnNumber = columnMeta.size(); + asyncWrite = taskConfig.getBool(Key.ASYNC_WRITE,false); + + int connectionsPerHost = taskConfig.getInt(Key.CONNECTIONS_PER_HOST,8); + int maxPendingPerConnection = taskConfig.getInt(Key.MAX_PENDING_CONNECTION,128); + PoolingOptions poolingOpts = new PoolingOptions() + .setConnectionsPerHost(HostDistance.LOCAL, connectionsPerHost, connectionsPerHost) + .setMaxRequestsPerConnection(HostDistance.LOCAL, maxPendingPerConnection) + .setNewConnectionThreshold(HostDistance.LOCAL, 100); + Cluster.Builder clusterBuilder = Cluster.builder().withPoolingOptions(poolingOpts); + if ((username != null) && !username.isEmpty()) { + clusterBuilder = clusterBuilder.withCredentials(username, password) + .withPort(Integer.valueOf(port)).addContactPoints(hosts.split(",")); + if (useSSL) { + clusterBuilder = clusterBuilder.withSSL(); + } + } else { + clusterBuilder = clusterBuilder.withPort(Integer.valueOf(port)) + .addContactPoints(hosts.split(",")); + } + cluster = clusterBuilder.build(); + session = cluster.connect(keyspace); + TableMetadata meta = cluster.getMetadata().getKeyspace(keyspace).getTable(table); + + Insert insertStmt = QueryBuilder.insertInto(table); + for( String colunmnName : columnMeta ) { + if( colunmnName.toLowerCase().equals(Key.WRITE_TIME) ) { + if( writeTimeCol != -1 ) { + throw DataXException + .asDataXException( + CassandraWriterErrorCode.CONF_ERROR, + "列配置信息有错误. 只能有一个时间戳列(writetime())"); + } + writeTimeCol = columnTypes.size(); + continue; + } + insertStmt.value(colunmnName,QueryBuilder.bindMarker()); + ColumnMetadata col = meta.getColumn(colunmnName); + if( col == null ) { + throw DataXException + .asDataXException( + CassandraWriterErrorCode.CONF_ERROR, + String.format( + "列配置信息有错误. 表中未找到列名 '%s' .", + colunmnName)); + } + columnTypes.add(col.getType()); + } + if(writeTimeCol != -1) { + insertStmt.using(timestamp(QueryBuilder.bindMarker())); + } + String cl = taskConfig.getString(Key.CONSITANCY_LEVEL); + if( cl != null && !cl.isEmpty() ) { + insertStmt.setConsistencyLevel(ConsistencyLevel.valueOf(cl)); + } else { + insertStmt.setConsistencyLevel(ConsistencyLevel.LOCAL_QUORUM); + } + + statement = session.prepare(insertStmt); + + if( batchSize > 1 ) { + if( asyncWrite ) { + unConfirmedWrite = new ArrayList(); + } else { + bufferedWrite = new ArrayList(); + } + } + + } + + @Override public void destroy() { + + } + } +} diff --git a/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterErrorCode.java b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterErrorCode.java new file mode 100644 index 00000000..dd31c7ab --- /dev/null +++ b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterErrorCode.java @@ -0,0 +1,35 @@ +package com.alibaba.datax.plugin.writer.cassandrawriter; + +import com.alibaba.datax.common.spi.ErrorCode; + +/** + * Created by mazhenlin on 2019/8/19. + */ +public enum CassandraWriterErrorCode implements ErrorCode { + CONF_ERROR("CassandraWriter-00", "配置错误."), + WRITE_DATA_ERROR("CassandraWriter-01", "写入数据时失败."), + ; + + private final String code; + private final String description; + + private CassandraWriterErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s].", this.code, this.description); + } +} diff --git a/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterHelper.java b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterHelper.java new file mode 100644 index 00000000..b68af281 --- /dev/null +++ b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterHelper.java @@ -0,0 +1,351 @@ +package com.alibaba.datax.plugin.writer.cassandrawriter; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONException; +import com.alibaba.fastjson.JSONObject; + +import com.datastax.driver.core.BoundStatement; +import com.datastax.driver.core.CodecRegistry; +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.DataType.Name; +import com.datastax.driver.core.Duration; +import com.datastax.driver.core.LocalDate; +import com.datastax.driver.core.TupleType; +import com.datastax.driver.core.TupleValue; +import com.datastax.driver.core.UDTValue; +import com.datastax.driver.core.UserType; +import com.datastax.driver.core.UserType.Field; +import com.google.common.base.Splitter; +import org.apache.commons.codec.binary.Base64; + +/** + * Created by mazhenlin on 2019/8/21. + */ +public class CassandraWriterHelper { + static CodecRegistry registry = new CodecRegistry(); + + public static Object parseFromString(String s, DataType sqlType ) throws Exception { + if (s == null || s.isEmpty()) { + if (sqlType.getName() == Name.ASCII || sqlType.getName() == Name.TEXT || + sqlType.getName() == Name.VARCHAR) { + return s; + } else { + return null; + } + } + switch (sqlType.getName()) { + case ASCII: + case TEXT: + case VARCHAR: + return s; + + case BLOB: + if (s.length() == 0) { + return new byte[0]; + } + byte[] byteArray = new byte[s.length() / 2]; + for (int i = 0; i < byteArray.length; i++) { + String subStr = s.substring(2 * i, 2 * i + 2); + byteArray[i] = ((byte) Integer.parseInt(subStr, 16)); + } + return ByteBuffer.wrap(byteArray); + + case BOOLEAN: + return Boolean.valueOf(s); + + case TINYINT: + return Byte.valueOf(s); + + case SMALLINT: + return Short.valueOf(s); + + case INT: + return Integer.valueOf(s); + + case BIGINT: + return Long.valueOf(s); + + case VARINT: + return new BigInteger(s, 10); + + case FLOAT: + return Float.valueOf(s); + + case DOUBLE: + return Double.valueOf(s); + + case DECIMAL: + return new BigDecimal(s); + + case DATE: { + String[] a = s.split("-"); + if (a.length != 3) { + throw new Exception(String.format("DATE类型数据 '%s' 格式不正确，必须为yyyy-mm-dd格式", s)); + } + return LocalDate.fromYearMonthDay(Integer.valueOf(a[0]), Integer.valueOf(a[1]), + Integer.valueOf(a[2])); + } + + case TIME: + return Long.valueOf(s); + + case TIMESTAMP: + return new Date(Long.valueOf(s)); + + case UUID: + case TIMEUUID: + return UUID.fromString(s); + + case INET: + String[] b = s.split("/"); + if (b.length < 2) { + return InetAddress.getByName(s); + } + byte[] addr = InetAddress.getByName(b[1]).getAddress(); + return InetAddress.getByAddress(b[0], addr); + + case DURATION: + return Duration.from(s); + + case LIST: + case MAP: + case SET: + case TUPLE: + case UDT: + Object jsonObject = JSON.parse(s); + return parseFromJson(jsonObject,sqlType); + + default: + throw DataXException.asDataXException(CassandraWriterErrorCode.CONF_ERROR, + "不支持您配置的列类型:" + sqlType + ", 请检查您的配置 或者 联系 管理员."); + + } // end switch + + } + + public static Object parseFromJson(Object jsonObject,DataType type) throws Exception { + if( jsonObject == null ) return null; + switch (type.getName()) { + case ASCII: + case TEXT: + case VARCHAR: + case BOOLEAN: + case TIME: + return jsonObject; + + case TINYINT: + return ((Number)jsonObject).byteValue(); + + case SMALLINT: + return ((Number)jsonObject).shortValue(); + + case INT: + return ((Number)jsonObject).intValue(); + + case BIGINT: + return ((Number)jsonObject).longValue(); + + case VARINT: + return new BigInteger(jsonObject.toString()); + + case FLOAT: + return ((Number)jsonObject).floatValue(); + + case DOUBLE: + return ((Number)jsonObject).doubleValue(); + + case DECIMAL: + return new BigDecimal(jsonObject.toString()); + + case BLOB: + return ByteBuffer.wrap(Base64.decodeBase64((String)jsonObject)); + + case DATE: + return LocalDate.fromMillisSinceEpoch(((Number)jsonObject).longValue()); + + case TIMESTAMP: + return new Date(((Number)jsonObject).longValue()); + + case DURATION: + return Duration.from(jsonObject.toString()); + + case UUID: + case TIMEUUID: + return UUID.fromString(jsonObject.toString()); + + case INET: + return InetAddress.getByName((String)jsonObject); + + case LIST: + List l = new ArrayList(); + for( Object o : (JSONArray)jsonObject ) { + l.add(parseFromJson(o,type.getTypeArguments().get(0))); + } + return l; + + case MAP: { + Map m = new HashMap(); + for (JSONObject.Entry e : ((JSONObject)jsonObject).entrySet()) { + Object k = parseFromString((String) e.getKey(), type.getTypeArguments().get(0)); + Object v = parseFromJson(e.getValue(), type.getTypeArguments().get(1)); + m.put(k,v); + } + return m; + } + + case SET: + Set s = new HashSet(); + for( Object o : (JSONArray)jsonObject ) { + s.add(parseFromJson(o,type.getTypeArguments().get(0))); + } + return s; + + case TUPLE: { + TupleValue t = ((TupleType) type).newValue(); + int j = 0; + for (Object e : (JSONArray)jsonObject) { + DataType eleType = ((TupleType) type).getComponentTypes().get(j); + t.set(j, parseFromJson(e, eleType), registry.codecFor(eleType).getJavaType()); + j++; + } + return t; + } + + case UDT: { + UDTValue t = ((UserType) type).newValue(); + UserType userType = t.getType(); + for (JSONObject.Entry e : ((JSONObject)jsonObject).entrySet()) { + DataType eleType = userType.getFieldType((String)e.getKey()); + t.set((String)e.getKey(), parseFromJson(e.getValue(), eleType), registry.codecFor(eleType).getJavaType()); + } + return t; + } + + } + return null; + } + + public static void setupColumn(BoundStatement ps, int pos, DataType sqlType, Column col) throws Exception { + if (col.getRawData() != null) { + switch (sqlType.getName()) { + case ASCII: + case TEXT: + case VARCHAR: + ps.setString(pos, col.asString()); + break; + + case BLOB: + ps.setBytes(pos, ByteBuffer.wrap(col.asBytes())); + break; + + case BOOLEAN: + ps.setBool(pos, col.asBoolean()); + break; + + case TINYINT: + ps.setByte(pos, col.asLong().byteValue()); + break; + + case SMALLINT: + ps.setShort(pos, col.asLong().shortValue()); + break; + + case INT: + ps.setInt(pos, col.asLong().intValue()); + break; + + case BIGINT: + ps.setLong(pos, col.asLong()); + break; + + case VARINT: + ps.setVarint(pos, col.asBigInteger()); + break; + + case FLOAT: + ps.setFloat(pos, col.asDouble().floatValue()); + break; + + case DOUBLE: + ps.setDouble(pos, col.asDouble()); + break; + + case DECIMAL: + ps.setDecimal(pos, col.asBigDecimal()); + break; + + case DATE: + ps.setDate(pos, LocalDate.fromMillisSinceEpoch(col.asDate().getTime())); + break; + + case TIME: + ps.setTime(pos, col.asLong()); + break; + + case TIMESTAMP: + ps.setTimestamp(pos, col.asDate()); + break; + + case UUID: + case TIMEUUID: + ps.setUUID(pos, UUID.fromString(col.asString())); + break; + + case INET: + ps.setInet(pos, InetAddress.getByName(col.asString())); + break; + + case DURATION: + ps.set(pos, Duration.from(col.asString()), Duration.class); + break; + + case LIST: + ps.setList(pos, (List) parseFromString(col.asString(), sqlType)); + break; + + case MAP: + ps.setMap(pos, (Map) parseFromString(col.asString(), sqlType)); + break; + + case SET: + ps.setSet(pos, (Set) parseFromString(col.asString(), sqlType)); + break; + + case TUPLE: + ps.setTupleValue(pos, (TupleValue) parseFromString(col.asString(), sqlType)); + break; + + case UDT: + ps.setUDTValue(pos, (UDTValue) parseFromString(col.asString(), sqlType)); + break; + + default: + throw DataXException.asDataXException(CassandraWriterErrorCode.CONF_ERROR, + "不支持您配置的列类型:" + sqlType + ", 请检查您的配置 或者 联系 管理员."); + + } // end switch + } else { + ps.setToNull(pos); + } + } + +} diff --git a/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/Key.java b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/Key.java new file mode 100644 index 00000000..755103a1 --- /dev/null +++ b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/Key.java @@ -0,0 +1,43 @@ +package com.alibaba.datax.plugin.writer.cassandrawriter; + +/** + * Created by mazhenlin on 2019/8/19. + */ +public class Key { + public final static String USERNAME = "username"; + public final static String PASSWORD = "password"; + + public final static String HOST = "host"; + public final static String PORT = "port"; + public final static String USESSL = "useSSL"; + + public final static String KEYSPACE = "keyspace"; + public final static String TABLE = "table"; + public final static String COLUMN = "column"; + public final static String WRITE_TIME = "writetime()"; + public final static String ASYNC_WRITE = "asyncWrite"; + public final static String CONSITANCY_LEVEL = "consistancyLevel"; + public final static String CONNECTIONS_PER_HOST = "connectionsPerHost"; + public final static String MAX_PENDING_CONNECTION = "maxPendingPerConnection"; + /** + * 异步写入的批次大小，默认1（不异步写入） + */ + public final static String BATCH_SIZE = "batchSize"; + + /** + * 每个列的名字 + */ + public static final String COLUMN_NAME = "name"; + /** + * 列分隔符 + */ + public static final String COLUMN_SPLITTER = "format"; + public static final String ELEMENT_SPLITTER = "splitter"; + public static final String ENTRY_SPLITTER = "entrySplitter"; + public static final String KV_SPLITTER = "kvSplitter"; + public static final String ELEMENT_CONFIG = "element"; + public static final String TUPLE_CONNECTOR = "_"; + public static final String KEY_CONFIG = "key"; + public static final String VALUE_CONFIG = "value"; + +} diff --git a/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings.properties b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings.properties new file mode 100644 index 00000000..607dd2e1 --- /dev/null +++ b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings.properties @@ -0,0 +1,2 @@ +errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. +errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_en_US.properties b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_en_US.properties new file mode 100644 index 00000000..4116518e --- /dev/null +++ b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_en_US.properties @@ -0,0 +1,2 @@ +errorcode.config_invalid_exception=Error in parameter configuration. +errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_ja_JP.properties b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_ja_JP.properties new file mode 100644 index 00000000..607dd2e1 --- /dev/null +++ b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_ja_JP.properties @@ -0,0 +1,2 @@ +errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. +errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_CN.properties b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_CN.properties new file mode 100644 index 00000000..607dd2e1 --- /dev/null +++ b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_CN.properties @@ -0,0 +1,2 @@ +errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. +errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_HK.properties b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_HK.properties new file mode 100644 index 00000000..607dd2e1 --- /dev/null +++ b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_HK.properties @@ -0,0 +1,2 @@ +errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. +errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_TW.properties b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_TW.properties new file mode 100644 index 00000000..607dd2e1 --- /dev/null +++ b/cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_TW.properties @@ -0,0 +1,2 @@ +errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. +errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 \ No newline at end of file diff --git a/cassandrawriter/src/main/resources/plugin.json b/cassandrawriter/src/main/resources/plugin.json new file mode 100644 index 00000000..661e6748 --- /dev/null +++ b/cassandrawriter/src/main/resources/plugin.json @@ -0,0 +1,7 @@ +{ + "name": "cassandrawriter", + "class": "com.alibaba.datax.plugin.writer.cassandrawriter.CassandraWriter", + "description": "useScene: prod. mechanism: use datax driver, execute insert sql.", + "developer": "alibaba" +} + diff --git a/cassandrawriter/src/main/resources/plugin_job_template.json b/cassandrawriter/src/main/resources/plugin_job_template.json new file mode 100644 index 00000000..d0952558 --- /dev/null +++ b/cassandrawriter/src/main/resources/plugin_job_template.json @@ -0,0 +1,15 @@ +{ + "name": "cassandrawriter", + "parameter": { + "username": "", + "password": "", + "host": "", + "port": "", + "useSSL": false, + "keyspace": "", + "table": "", + "column": [ + "c1","c2","c3" + ] + } +} \ No newline at end of file diff --git a/package.xml b/package.xml index 8b7c72f7..42185f54 100755 --- a/package.xml +++ b/package.xml @@ -166,6 +166,13 @@ datax + + cassandrareader/target/datax/ + + **/*.* + + datax + @@ -343,5 +350,12 @@ datax + + cassandrawriter/target/datax/ + + **/*.* + + datax + diff --git a/pom.xml b/pom.xml index 50ee2080..d7fd24f1 100755 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,7 @@ hbase11xreader hbase094xreader opentsdbreader + cassandrareader mysqlwriter @@ -89,6 +90,7 @@ tsdbwriter adbpgwriter gdbwriter + cassandrawriter plugin-rdbms-util From 0af1933802cfa3ba13050daad9205715cc484f35 Mon Sep 17 00:00:00 2001 From: asdf2014 Date: Fri, 8 Nov 2019 15:08:16 +0800 Subject: [PATCH 2/4] Bump jdk to 1.8 --- adbpgwriter/pom.xml | 4 ++-- adswriter/pom.xml | 4 ++-- cassandrareader/pom.xml | 4 ++-- cassandrawriter/pom.xml | 4 ++-- common/pom.xml | 4 ++-- core/pom.xml | 4 ++-- drdsreader/pom.xml | 4 ++-- drdswriter/pom.xml | 4 ++-- elasticsearchwriter/pom.xml | 4 ++-- ftpreader/pom.xml | 4 ++-- ftpwriter/pom.xml | 4 ++-- gdbwriter/pom.xml | 4 ++-- hbase094xreader/pom.xml | 10 ++++++++-- hbase094xwriter/pom.xml | 10 ++++++++-- hbase11xreader/pom.xml | 10 ++++++++-- hbase11xsqlreader/pom.xml | 8 ++++++-- hbase11xsqlwriter/pom.xml | 10 ++++++++-- hbase11xwriter/pom.xml | 10 ++++++++-- hbase20xsqlreader/pom.xml | 4 ++-- hbase20xsqlwriter/pom.xml | 4 ++-- hdfsreader/pom.xml | 10 ++++++++-- hdfswriter/pom.xml | 4 ++-- mongodbreader/pom.xml | 4 ++-- mongodbwriter/pom.xml | 4 ++-- mysqlreader/pom.xml | 4 ++-- mysqlwriter/pom.xml | 4 ++-- ocswriter/pom.xml | 4 ++-- odpsreader/pom.xml | 4 ++-- odpswriter/pom.xml | 4 ++-- opentsdbreader/pom.xml | 4 ++-- oraclereader/pom.xml | 4 ++-- oraclewriter/pom.xml | 4 ++-- ossreader/pom.xml | 4 ++-- osswriter/pom.xml | 4 ++-- otsreader/pom.xml | 4 ++-- otsstreamreader/pom.xml | 4 ++-- otswriter/pom.xml | 4 ++-- pom.xml | 5 +++-- postgresqlreader/pom.xml | 4 ++-- postgresqlwriter/pom.xml | 4 ++-- rdbmsreader/pom.xml | 4 ++-- rdbmswriter/pom.xml | 4 ++-- sqlserverreader/pom.xml | 4 ++-- sqlserverwriter/pom.xml | 4 ++-- streamreader/pom.xml | 4 ++-- streamwriter/pom.xml | 4 ++-- transformer/pom.xml | 4 ++-- tsdbwriter/pom.xml | 4 ++-- txtfilereader/pom.xml | 4 ++-- txtfilewriter/pom.xml | 4 ++-- 50 files changed, 141 insertions(+), 100 deletions(-) diff --git a/adbpgwriter/pom.xml b/adbpgwriter/pom.xml index 094dbd9c..c2800c7b 100644 --- a/adbpgwriter/pom.xml +++ b/adbpgwriter/pom.xml @@ -84,8 +84,8 @@ maven-compiler-plugin - 1.8 - 1.8 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/adswriter/pom.xml b/adswriter/pom.xml index 90be43e8..41271de9 100644 --- a/adswriter/pom.xml +++ b/adswriter/pom.xml @@ -94,8 +94,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/cassandrareader/pom.xml b/cassandrareader/pom.xml index 4acc5e3a..8aa38936 100644 --- a/cassandrareader/pom.xml +++ b/cassandrareader/pom.xml @@ -104,8 +104,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/cassandrawriter/pom.xml b/cassandrawriter/pom.xml index 01dfa62f..7d7cfbd7 100644 --- a/cassandrawriter/pom.xml +++ b/cassandrawriter/pom.xml @@ -96,8 +96,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/common/pom.xml b/common/pom.xml index 6cce789f..1a57cccd 100755 --- a/common/pom.xml +++ b/common/pom.xml @@ -65,8 +65,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/core/pom.xml b/core/pom.xml index 5582d943..ea3d53e7 100755 --- a/core/pom.xml +++ b/core/pom.xml @@ -140,8 +140,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/drdsreader/pom.xml b/drdsreader/pom.xml index 2f890eca..71c7108d 100755 --- a/drdsreader/pom.xml +++ b/drdsreader/pom.xml @@ -54,8 +54,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/drdswriter/pom.xml b/drdswriter/pom.xml index fc852ff8..6a361760 100755 --- a/drdswriter/pom.xml +++ b/drdswriter/pom.xml @@ -54,8 +54,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/elasticsearchwriter/pom.xml b/elasticsearchwriter/pom.xml index 39ee97e0..2a246805 100644 --- a/elasticsearchwriter/pom.xml +++ b/elasticsearchwriter/pom.xml @@ -60,8 +60,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/ftpreader/pom.xml b/ftpreader/pom.xml index c06c6542..7778d491 100755 --- a/ftpreader/pom.xml +++ b/ftpreader/pom.xml @@ -63,8 +63,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/ftpwriter/pom.xml b/ftpwriter/pom.xml index 1fd766c9..69ec4a07 100644 --- a/ftpwriter/pom.xml +++ b/ftpwriter/pom.xml @@ -65,8 +65,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/gdbwriter/pom.xml b/gdbwriter/pom.xml index a5d3f319..584a1c09 100644 --- a/gdbwriter/pom.xml +++ b/gdbwriter/pom.xml @@ -74,8 +74,8 @@ maven-compiler-plugin - 1.8 - 1.8 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/hbase094xreader/pom.xml b/hbase094xreader/pom.xml index 9170536c..d996ba1a 100644 --- a/hbase094xreader/pom.xml +++ b/hbase094xreader/pom.xml @@ -39,6 +39,12 @@ org.apache.hbase hbase 0.94.27 + + + jdk.tools + jdk.tools + + org.apache.hadoop @@ -66,8 +72,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/hbase094xwriter/pom.xml b/hbase094xwriter/pom.xml index f3f403c1..47e6db4c 100644 --- a/hbase094xwriter/pom.xml +++ b/hbase094xwriter/pom.xml @@ -42,6 +42,12 @@ org.apache.hbase hbase 0.94.27 + + + jdk.tools + jdk.tools + + org.apache.hadoop @@ -79,8 +85,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/hbase11xreader/pom.xml b/hbase11xreader/pom.xml index 537fd625..9a82b852 100644 --- a/hbase11xreader/pom.xml +++ b/hbase11xreader/pom.xml @@ -37,6 +37,12 @@ org.apache.hbase hbase-client ${hbase.version} + + + jdk.tools + jdk.tools + + org.apache.hbase @@ -86,8 +92,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/hbase11xsqlreader/pom.xml b/hbase11xsqlreader/pom.xml index 6ea727c0..bf9e73f1 100644 --- a/hbase11xsqlreader/pom.xml +++ b/hbase11xsqlreader/pom.xml @@ -37,6 +37,10 @@ servlet-api javax.servlet + + jdk.tools + jdk.tools + @@ -80,8 +84,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/hbase11xsqlwriter/pom.xml b/hbase11xsqlwriter/pom.xml index 3b75167e..bc4b61cd 100644 --- a/hbase11xsqlwriter/pom.xml +++ b/hbase11xsqlwriter/pom.xml @@ -48,6 +48,12 @@ org.apache.phoenix phoenix-core ${phoenix.version} + + + jdk.tools + jdk.tools + + org.apache.phoenix @@ -120,8 +126,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/hbase11xwriter/pom.xml b/hbase11xwriter/pom.xml index 797bef32..9b86c27d 100644 --- a/hbase11xwriter/pom.xml +++ b/hbase11xwriter/pom.xml @@ -41,6 +41,12 @@ org.apache.hbase hbase-client ${hbase.version} + + + jdk.tools + jdk.tools + + org.apache.hbase @@ -95,8 +101,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/hbase20xsqlreader/pom.xml b/hbase20xsqlreader/pom.xml index 56804b7b..ec1c3419 100644 --- a/hbase20xsqlreader/pom.xml +++ b/hbase20xsqlreader/pom.xml @@ -82,8 +82,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/hbase20xsqlwriter/pom.xml b/hbase20xsqlwriter/pom.xml index 9d363f63..690bc95e 100644 --- a/hbase20xsqlwriter/pom.xml +++ b/hbase20xsqlwriter/pom.xml @@ -76,8 +76,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/hdfsreader/pom.xml b/hdfsreader/pom.xml index b3c91a5e..5d07dc25 100644 --- a/hdfsreader/pom.xml +++ b/hdfsreader/pom.xml @@ -71,6 +71,12 @@ org.apache.hive hive-service ${hive.version} + + + jdk.tools + jdk.tools + + org.apache.hive @@ -97,8 +103,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/hdfswriter/pom.xml b/hdfswriter/pom.xml index 574c23b6..15b3780a 100644 --- a/hdfswriter/pom.xml +++ b/hdfswriter/pom.xml @@ -106,8 +106,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/mongodbreader/pom.xml b/mongodbreader/pom.xml index fec3bfd8..97356dc7 100644 --- a/mongodbreader/pom.xml +++ b/mongodbreader/pom.xml @@ -55,8 +55,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/mongodbwriter/pom.xml b/mongodbwriter/pom.xml index 0360db2a..ac6d9394 100644 --- a/mongodbwriter/pom.xml +++ b/mongodbwriter/pom.xml @@ -59,8 +59,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/mysqlreader/pom.xml b/mysqlreader/pom.xml index aa459122..08183272 100755 --- a/mysqlreader/pom.xml +++ b/mysqlreader/pom.xml @@ -52,8 +52,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/mysqlwriter/pom.xml b/mysqlwriter/pom.xml index 58b93179..11618022 100755 --- a/mysqlwriter/pom.xml +++ b/mysqlwriter/pom.xml @@ -50,8 +50,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/ocswriter/pom.xml b/ocswriter/pom.xml index 34e49cab..51e75b74 100644 --- a/ocswriter/pom.xml +++ b/ocswriter/pom.xml @@ -56,8 +56,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} 3.2 diff --git a/odpsreader/pom.xml b/odpsreader/pom.xml index 9204d908..e7244917 100755 --- a/odpsreader/pom.xml +++ b/odpsreader/pom.xml @@ -115,8 +115,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/odpswriter/pom.xml b/odpswriter/pom.xml index 38672cff..b37a03d1 100755 --- a/odpswriter/pom.xml +++ b/odpswriter/pom.xml @@ -78,8 +78,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/opentsdbreader/pom.xml b/opentsdbreader/pom.xml index bcbf8414..aa3461d8 100644 --- a/opentsdbreader/pom.xml +++ b/opentsdbreader/pom.xml @@ -126,8 +126,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/oraclereader/pom.xml b/oraclereader/pom.xml index fc91ca29..ae8e06fa 100755 --- a/oraclereader/pom.xml +++ b/oraclereader/pom.xml @@ -56,8 +56,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/oraclewriter/pom.xml b/oraclewriter/pom.xml index 90104049..95b78caf 100755 --- a/oraclewriter/pom.xml +++ b/oraclewriter/pom.xml @@ -53,8 +53,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/ossreader/pom.xml b/ossreader/pom.xml index de6fe3ad..1feb42ff 100755 --- a/ossreader/pom.xml +++ b/ossreader/pom.xml @@ -58,8 +58,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/osswriter/pom.xml b/osswriter/pom.xml index a5cb76cf..90d84c10 100644 --- a/osswriter/pom.xml +++ b/osswriter/pom.xml @@ -52,8 +52,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/otsreader/pom.xml b/otsreader/pom.xml index f6f7673b..bd017423 100644 --- a/otsreader/pom.xml +++ b/otsreader/pom.xml @@ -47,8 +47,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/otsstreamreader/pom.xml b/otsstreamreader/pom.xml index dca4de23..9e5e476d 100644 --- a/otsstreamreader/pom.xml +++ b/otsstreamreader/pom.xml @@ -61,8 +61,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/otswriter/pom.xml b/otswriter/pom.xml index 018e011f..8677c8ab 100644 --- a/otswriter/pom.xml +++ b/otswriter/pom.xml @@ -47,8 +47,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/pom.xml b/pom.xml index d7fd24f1..ece9491e 100755 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,7 @@ pom + 1.8 0.0.1-SNAPSHOT 3.3.2 1.10 @@ -197,8 +198,8 @@ maven-compiler-plugin 2.3.2 - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/postgresqlreader/pom.xml b/postgresqlreader/pom.xml index 48a3c615..e7f90325 100755 --- a/postgresqlreader/pom.xml +++ b/postgresqlreader/pom.xml @@ -56,8 +56,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/postgresqlwriter/pom.xml b/postgresqlwriter/pom.xml index b90cf307..c3240e04 100755 --- a/postgresqlwriter/pom.xml +++ b/postgresqlwriter/pom.xml @@ -53,8 +53,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/rdbmsreader/pom.xml b/rdbmsreader/pom.xml index 34fbbada..e79d86f7 100755 --- a/rdbmsreader/pom.xml +++ b/rdbmsreader/pom.xml @@ -70,8 +70,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/rdbmswriter/pom.xml b/rdbmswriter/pom.xml index bb20a74c..19461960 100755 --- a/rdbmswriter/pom.xml +++ b/rdbmswriter/pom.xml @@ -71,8 +71,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/sqlserverreader/pom.xml b/sqlserverreader/pom.xml index 0ec2a609..5372a057 100755 --- a/sqlserverreader/pom.xml +++ b/sqlserverreader/pom.xml @@ -48,8 +48,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/sqlserverwriter/pom.xml b/sqlserverwriter/pom.xml index f4879561..d2b1eea1 100644 --- a/sqlserverwriter/pom.xml +++ b/sqlserverwriter/pom.xml @@ -50,8 +50,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/streamreader/pom.xml b/streamreader/pom.xml index f7b12d50..dc754d9a 100755 --- a/streamreader/pom.xml +++ b/streamreader/pom.xml @@ -44,8 +44,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/streamwriter/pom.xml b/streamwriter/pom.xml index 58b29471..4a987fac 100755 --- a/streamwriter/pom.xml +++ b/streamwriter/pom.xml @@ -39,8 +39,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/transformer/pom.xml b/transformer/pom.xml index 8f3b7aee..cbfab8ea 100644 --- a/transformer/pom.xml +++ b/transformer/pom.xml @@ -38,8 +38,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/tsdbwriter/pom.xml b/tsdbwriter/pom.xml index d74776af..497264c0 100644 --- a/tsdbwriter/pom.xml +++ b/tsdbwriter/pom.xml @@ -106,8 +106,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/txtfilereader/pom.xml b/txtfilereader/pom.xml index f1c79db7..30671d76 100755 --- a/txtfilereader/pom.xml +++ b/txtfilereader/pom.xml @@ -49,8 +49,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} diff --git a/txtfilewriter/pom.xml b/txtfilewriter/pom.xml index 7d6489ee..30a6cf61 100755 --- a/txtfilewriter/pom.xml +++ b/txtfilewriter/pom.xml @@ -50,8 +50,8 @@ maven-compiler-plugin - 1.6 - 1.6 + ${jdk-version} + ${jdk-version} ${project-sourceEncoding} From dd19fd4332289c858042de30f86467fc14574d38 Mon Sep 17 00:00:00 2001 From: asdf2014 Date: Fri, 8 Nov 2019 16:52:39 +0800 Subject: [PATCH 3/4] Add TSDB Reader --- pom.xml | 1 + tsdbreader/doc/tsdbreader.md | 587 ++++++++++++++++++ tsdbreader/pom.xml | 146 +++++ tsdbreader/src/main/assembly/package.xml | 35 ++ .../plugin/reader/tsdbreader/Constant.java | 29 + .../datax/plugin/reader/tsdbreader/Key.java | 36 ++ .../plugin/reader/tsdbreader/TSDBReader.java | 320 ++++++++++ .../tsdbreader/TSDBReaderErrorCode.java | 40 ++ .../tsdbreader/conn/Connection4TSDB.java | 88 +++ .../conn/DataPoint4MultiFieldsTSDB.java | 68 ++ .../tsdbreader/conn/DataPoint4TSDB.java | 68 ++ .../conn/MultiFieldQueryResult.java | 64 ++ .../reader/tsdbreader/conn/QueryResult.java | 64 ++ .../tsdbreader/conn/TSDBConnection.java | 94 +++ .../reader/tsdbreader/conn/TSDBDump.java | 318 ++++++++++ .../reader/tsdbreader/util/HttpUtils.java | 67 ++ .../reader/tsdbreader/util/TSDBUtils.java | 68 ++ .../reader/tsdbreader/util/TimeUtils.java | 38 ++ tsdbreader/src/main/resources/plugin.json | 10 + .../main/resources/plugin_job_template.json | 29 + .../tsdbreader/conn/TSDBConnectionTest.java | 30 + .../plugin/reader/tsdbreader/util/Const.java | 17 + .../reader/tsdbreader/util/HttpUtilsTest.java | 39 ++ .../reader/tsdbreader/util/TimeUtilsTest.java | 33 + 24 files changed, 2289 insertions(+) create mode 100644 tsdbreader/doc/tsdbreader.md create mode 100644 tsdbreader/pom.xml create mode 100755 tsdbreader/src/main/assembly/package.xml create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Constant.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Key.java create mode 100755 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReader.java create mode 100755 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReaderErrorCode.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/Connection4TSDB.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/DataPoint4MultiFieldsTSDB.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/DataPoint4TSDB.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/MultiFieldQueryResult.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/QueryResult.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnection.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBDump.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtils.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TSDBUtils.java create mode 100644 tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TimeUtils.java create mode 100755 tsdbreader/src/main/resources/plugin.json create mode 100644 tsdbreader/src/main/resources/plugin_job_template.json create mode 100644 tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnectionTest.java create mode 100644 tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/Const.java create mode 100644 tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtilsTest.java create mode 100644 tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TimeUtilsTest.java diff --git a/pom.xml b/pom.xml index ece9491e..9463f2bf 100755 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,7 @@ rdbmsreader hbase11xreader hbase094xreader + tsdbreader opentsdbreader cassandrareader diff --git a/tsdbreader/doc/tsdbreader.md b/tsdbreader/doc/tsdbreader.md new file mode 100644 index 00000000..557dcc51 --- /dev/null +++ b/tsdbreader/doc/tsdbreader.md @@ -0,0 +1,587 @@ + +# TSDBReader 插件文档 + +___ + + +## 1 快速介绍 + +TSDBReader 插件实现了从阿里云 TSDB 读取数据。阿里云时间序列数据库 ( **T**ime **S**eries **D**ata**b**ase , 简称 TSDB) 是一种集时序数据高效读写，压缩存储，实时计算能力为一体的数据库服务，可广泛应用于物联网和互联网领域，实现对设备及业务服务的实时监控，实时预测告警。详见 TSDB 的阿里云[官网](https://cn.aliyun.com/product/hitsdb)。 + + + +## 2 实现原理 + +在底层实现上，TSDBReader 通过 HTTP 请求链接到 阿里云 TSDB 实例，利用 `/api/query` 或者 `/api/mquery` 接口将数据点扫描出来（更多细节详见：[时序数据库 TSDB - HTTP API 概览](https://help.aliyun.com/document_detail/63557.html)）。而整个同步的过程，是通过时间线和查询时间线范围进行切分。 + + + +## 3 功能说明 + +### 3.1 配置样例 + +* 配置一个从 阿里云 TSDB 数据库同步抽取数据到本地的作业，并以**时序数据**的格式输出： + +时序数据样例： +```json +{"metric":"m","tags":{"app":"a19","cluster":"c5","group":"g10","ip":"i999","zone":"z1"},"timestamp":1546272263,"value":1} +``` + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tsdbreader", + "parameter": { + "sinkDbType": "TSDB", + "endpoint": "http://localhost:8242", + "column": [ + "m" + ], + "splitIntervalMs": 60000, + "beginDateTime": "2019-01-01 00:00:00", + "endDateTime": "2019-01-01 01:00:00" + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "encoding": "UTF-8", + "print": true + } + } + } + ], + "setting": { + "speed": { + "channel": 3 + } + } + } +} +``` + +* 配置一个从 阿里云 TSDB 数据库同步抽取数据到本地的作业，并以**关系型数据**的格式输出： + +关系型数据样例： +```txt +m 1546272125 a1 c1 g2 i3021 z4 1.0 +``` + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tsdbreader", + "parameter": { + "sinkDbType": "RDB", + "endpoint": "http://localhost:8242", + "column": [ + "__metric__", + "__ts__", + "app", + "cluster", + "group", + "ip", + "zone", + "__value__" + ], + "metric": [ + "m" + ], + "splitIntervalMs": 60000, + "beginDateTime": "2019-01-01 00:00:00", + "endDateTime": "2019-01-01 01:00:00" + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "encoding": "UTF-8", + "print": true + } + } + } + ], + "setting": { + "speed": { + "channel": 3 + } + } + } +} +``` + +* 配置一个从 阿里云 TSDB 数据库同步抽取**单值**数据到 ADB 的作业： + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tsdbreader", + "parameter": { + "sinkDbType": "RDB", + "endpoint": "http://localhost:8242", + "column": [ + "__metric__", + "__ts__", + "app", + "cluster", + "group", + "ip", + "zone", + "__value__" + ], + "metric": [ + "m" + ], + "splitIntervalMs": 60000, + "beginDateTime": "2019-01-01 00:00:00", + "endDateTime": "2019-01-01 01:00:00" + } + }, + "writer": { + "name": "adswriter", + "parameter": { + "username": "******", + "password": "******", + "column": [ + "`metric`", + "`ts`", + "`app`", + "`cluster`", + "`group`", + "`ip`", + "`zone`", + "`value`" + ], + "url": "http://localhost:3306", + "schema": "datax_test", + "table": "datax_test", + "writeMode": "insert", + "opIndex": "0", + "batchSize": "2" + } + } + } + ], + "setting": { + "speed": { + "channel": 3 + } + } + } +} +``` + +* 配置一个从 阿里云 TSDB 数据库同步抽取**多值**数据到 ADB 的作业： + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tsdbreader", + "parameter": { + "sinkDbType": "RDB", + "endpoint": "http://localhost:8242", + "column": [ + "__metric__", + "__ts__", + "app", + "cluster", + "group", + "ip", + "zone", + "load", + "memory", + "cpu" + ], + "metric": [ + "m_field" + ], + "field": { + "m_field": [ + "load", + "memory", + "cpu" + ] + }, + "splitIntervalMs": 60000, + "beginDateTime": "2019-01-01 00:00:00", + "endDateTime": "2019-01-01 01:00:00" + } + }, + "writer": { + "name": "adswriter", + "parameter": { + "username": "******", + "password": "******", + "column": [ + "`metric`", + "`ts`", + "`app`", + "`cluster`", + "`group`", + "`ip`", + "`zone`", + "`load`", + "`memory`", + "`cpu`" + ], + "url": "http://localhost:3306", + "schema": "datax_test", + "table": "datax_test_multi_field", + "writeMode": "insert", + "opIndex": "0", + "batchSize": "2" + } + } + } + ], + "setting": { + "speed": { + "channel": 3 + } + } + } +} +``` + +* 配置一个从 阿里云 TSDB 数据库同步抽取**单值**数据到 ADB 的作业，并指定过滤部分时间线： + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tsdbreader", + "parameter": { + "sinkDbType": "RDB", + "endpoint": "http://localhost:8242", + "column": [ + "__metric__", + "__ts__", + "app", + "cluster", + "group", + "ip", + "zone", + "__value__" + ], + "metric": [ + "m" + ], + "tag": { + "m": { + "app": "a1", + "cluster": "c1" + } + }, + "splitIntervalMs": 60000, + "beginDateTime": "2019-01-01 00:00:00", + "endDateTime": "2019-01-01 01:00:00" + } + }, + "writer": { + "name": "adswriter", + "parameter": { + "username": "******", + "password": "******", + "column": [ + "`metric`", + "`ts`", + "`app`", + "`cluster`", + "`group`", + "`ip`", + "`zone`", + "`value`" + ], + "url": "http://localhost:3306", + "schema": "datax_test", + "table": "datax_test", + "writeMode": "insert", + "opIndex": "0", + "batchSize": "2" + } + } + } + ], + "setting": { + "speed": { + "channel": 3 + } + } + } +} +``` + +* 配置一个从 阿里云 TSDB 数据库同步抽取**多值**数据到 ADB 的作业，并指定过滤部分时间线： + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tsdbreader", + "parameter": { + "sinkDbType": "RDB", + "endpoint": "http://localhost:8242", + "column": [ + "__metric__", + "__ts__", + "app", + "cluster", + "group", + "ip", + "zone", + "load", + "memory", + "cpu" + ], + "metric": [ + "m_field" + ], + "field": { + "m_field": [ + "load", + "memory", + "cpu" + ] + }, + "tag": { + "m_field": { + "ip": "i999" + } + }, + "splitIntervalMs": 60000, + "beginDateTime": "2019-01-01 00:00:00", + "endDateTime": "2019-01-01 01:00:00" + } + }, + "writer": { + "name": "adswriter", + "parameter": { + "username": "******", + "password": "******", + "column": [ + "`metric`", + "`ts`", + "`app`", + "`cluster`", + "`group`", + "`ip`", + "`zone`", + "`load`", + "`memory`", + "`cpu`" + ], + "url": "http://localhost:3306", + "schema": "datax_test", + "table": "datax_test_multi_field", + "writeMode": "insert", + "opIndex": "0", + "batchSize": "2" + } + } + } + ], + "setting": { + "speed": { + "channel": 3 + } + } + } +} +``` + +* 配置一个从 阿里云 TSDB 数据库同步抽取**单值**数据到另一个 阿里云 TSDB 数据库 的作业： + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tsdbreader", + "parameter": { + "sinkDbType": "TSDB", + "endpoint": "http://localhost:8242", + "column": [ + "m" + ], + "splitIntervalMs": 60000, + "beginDateTime": "2019-01-01 00:00:00", + "endDateTime": "2019-01-01 01:00:00" + } + }, + "writer": { + "name": "tsdbwriter", + "parameter": { + "endpoint": "http://localhost:8240" + } + } + } + ], + "setting": { + "speed": { + "channel": 3 + } + } + } +} +``` + +* 配置一个从 阿里云 TSDB 数据库同步抽取**多值**数据到另一个 阿里云 TSDB 数据库 的作业： + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tsdbreader", + "parameter": { + "sinkDbType": "TSDB", + "endpoint": "http://localhost:8242", + "column": [ + "m_field" + ], + "field": { + "m_field": [ + "load", + "memory", + "cpu" + ] + }, + "splitIntervalMs": 60000, + "beginDateTime": "2019-01-01 00:00:00", + "endDateTime": "2019-01-01 01:00:00" + } + }, + "writer": { + "name": "tsdbwriter", + "parameter": { + "multiField": true, + "endpoint": "http://localhost:8240" + } + } + } + ], + "setting": { + "speed": { + "channel": 3 + } + } + } +} +``` + + + + + + +### 3.2 参数说明 + +* **name** + * 描述：本插件的名称 + * 必选：是 + * 默认值：tsdbreader + +* **parameter** + * **sinkDbType** + * 描述：目标数据库的类型 + * 必选：否 + * 默认值：TSDB + * 注意：目前支持 TSDB 和 RDB 两个取值。其中，TSDB 包括 阿里云 TSDB、OpenTSDB、InfluxDB、Prometheus 和 TimeScale。RDB 包括 ADB、MySQL、Oracle、PostgreSQL 和 DRDS 等。 + + * **endpoint** + * 描述：阿里云 TSDB 的 HTTP 连接地址 + * 必选：是 + * 格式：http://IP:Port + * 默认值：无 + + * **column** + * 描述：TSDB 场景下：数据迁移任务需要迁移的 Metric 列表；RDB 场景下：映射到关系型数据库中的表字段，且增加 `__metric__`、`__ts__` 和 `__value__` 三个字段，其中 `__metric__` 用于映射度量字段，`__ts__` 用于映射 timestamp 字段，而 `__value__` 仅适用于单值场景，用于映射度量值，多值场景下，直接指定 field 字段即可 + * 必选：是 + * 默认值：无 + + * **metric** + * 描述：仅适用于 RDB 场景下，表示数据迁移任务需要迁移的 Metric 列表 + * 必选：否 + * 默认值：无 + + * **field** + * 描述：仅适用于多值场景下，表示数据迁移任务需要迁移的 Field 列表 + * 必选：否 + * 默认值：无 + + * **tag** + * 描述：数据迁移任务需要迁移的 TagK 和 TagV，用于进一步过滤时间线 + * 必选：否 + * 默认值：无 + + * **splitIntervalMs** + * 描述：用于 DataX 内部切分 Task，每个 Task 只查询一小部分的时间段 + * 必选：是 + * 默认值：无 + * 注意：单位是 ms 毫秒 + + +* **beginDateTime** + * 描述：和 endDateTime 配合使用，用于指定哪个时间段内的数据点，需要被迁移 + * 必选：是 + * 格式：`yyyy-MM-dd HH:mm:ss` + * 默认值：无 + * 注意：指定起止时间会自动忽略分钟和秒，转为整点时刻，例如 2019-4-18 的 [3:35, 4:55) 会被转为 [3:00, 4:00) + +* **endDateTime** + * 描述：和 beginDateTime 配合使用，用于指定哪个时间段内的数据点，需要被迁移 + * 必选：是 + * 格式：`yyyy-MM-dd HH:mm:ss` + * 默认值：无 + * 注意：指定起止时间会自动忽略分钟和秒，转为整点时刻，例如 2019-4-18 的 [3:35, 4:55) 会被转为 [3:00, 4:00) + + + + +### 3.3 类型转换 + +| DataX 内部类型 | TSDB 数据类型 | +| -------------- | ------------------------------------------------------------ | +| String | TSDB 数据点序列化字符串，包括 timestamp、metric、tags、fields 和 value | + + + + + + +## 4 约束限制 + +### 4.2 如果存在某一个 Metric 下在一个小时范围内的数据量过大，可能需要通过 `-j` 参数调整 JVM 内存大小 + +考虑到下游 Writer 如果写入速度不及 TSDB Reader 的查询数据，可能会存在积压的情况，因此需要适当地调整 JVM 参数。以"从 阿里云 TSDB 数据库同步抽取数据到本地的作业"为例，启动命令如下： + +```bash + python datax/bin/datax.py tsdb2stream.json -j "-Xms4096m -Xmx4096m" +``` + + + +### 4.3 指定起止时间会自动被转为整点时刻 + +指定起止时间会自动被转为整点时刻，例如 2019-4-18 的 `[3:35, 3:55)` 会被转为 `[3:00, 4:00)` + + + diff --git a/tsdbreader/pom.xml b/tsdbreader/pom.xml new file mode 100644 index 00000000..b9a45985 --- /dev/null +++ b/tsdbreader/pom.xml @@ -0,0 +1,146 @@ + + + 4.0.0 + + + com.alibaba.datax + datax-all + 0.0.1-SNAPSHOT + + + tsdbreader + tsdbreader + jar + + + UTF-8 + + + 3.3.2 + + + 4.4 + 2.4 + + + 1.2.28 + + + 4.12 + + + 2.9.9 + + + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + fastjson + com.alibaba + + + commons-math3 + org.apache.commons + + + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + commons-io + commons-io + ${commons-io.version} + + + org.apache.httpcomponents + fluent-hc + ${httpclient.version} + + + + + com.alibaba + fastjson + ${fastjson.version} + + + + + joda-time + joda-time + ${joda-time.version} + + + + + junit + junit + ${junit4.version} + test + + + + + + + + maven-compiler-plugin + + ${jdk-version} + ${jdk-version} + ${project-sourceEncoding} + + + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + diff --git a/tsdbreader/src/main/assembly/package.xml b/tsdbreader/src/main/assembly/package.xml new file mode 100755 index 00000000..b76f2aba --- /dev/null +++ b/tsdbreader/src/main/assembly/package.xml @@ -0,0 +1,35 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/reader/tsdbreader + + + target/ + + tsdbreader-0.0.1-SNAPSHOT.jar + + plugin/reader/tsdbreader + + + + + + false + plugin/reader/tsdbreader/libs + runtime + + + diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Constant.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Constant.java new file mode 100644 index 00000000..e42dedc0 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Constant.java @@ -0,0 +1,29 @@ +package com.alibaba.datax.plugin.reader.tsdbreader; + +import java.util.HashSet; +import java.util.Set; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：Constant + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public final class Constant { + + static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss"; + + public static final String METRIC_SPECIFY_KEY = "__metric__"; + public static final String TS_SPECIFY_KEY = "__ts__"; + public static final String VALUE_SPECIFY_KEY = "__value__"; + + static final Set MUST_CONTAINED_SPECIFY_KEYS = new HashSet<>(); + + static { + MUST_CONTAINED_SPECIFY_KEYS.add(METRIC_SPECIFY_KEY); + MUST_CONTAINED_SPECIFY_KEYS.add(TS_SPECIFY_KEY); + // __value__ 在多值场景下，可以不指定 + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Key.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Key.java new file mode 100644 index 00000000..14ee7e41 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Key.java @@ -0,0 +1,36 @@ +package com.alibaba.datax.plugin.reader.tsdbreader; + +import java.util.HashSet; +import java.util.Set; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：Key + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public class Key { + + // TSDB for OpenTSDB / InfluxDB / TimeScale / Prometheus etc. + // RDB for MySQL / ADB etc. + static final String SINK_DB_TYPE = "sinkDbType"; + static final String ENDPOINT = "endpoint"; + static final String COLUMN = "column"; + static final String METRIC = "metric"; + static final String FIELD = "field"; + static final String TAG = "tag"; + static final String INTERVAL_DATE_TIME = "splitIntervalMs"; + static final String BEGIN_DATE_TIME = "beginDateTime"; + static final String END_DATE_TIME = "endDateTime"; + + static final Integer INTERVAL_DATE_TIME_DEFAULT_VALUE = 60; + static final String TYPE_DEFAULT_VALUE = "TSDB"; + static final Set TYPE_SET = new HashSet<>(); + + static { + TYPE_SET.add("TSDB"); + TYPE_SET.add("RDB"); + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReader.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReader.java new file mode 100755 index 00000000..04b931c7 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReader.java @@ -0,0 +1,320 @@ +package com.alibaba.datax.plugin.reader.tsdbreader; + +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.common.spi.Reader; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.plugin.reader.tsdbreader.conn.TSDBConnection; +import com.alibaba.datax.plugin.reader.tsdbreader.util.TimeUtils; +import com.alibaba.fastjson.JSON; +import org.apache.commons.lang3.StringUtils; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：TSDB Reader + * + * @author Benedict Jin + * @since 2019-10-21 + */ +@SuppressWarnings("unused") +public class TSDBReader extends Reader { + + public static class Job extends Reader.Job { + + private static final Logger LOG = LoggerFactory.getLogger(Job.class); + + private Configuration originalConfig; + + @Override + public void init() { + this.originalConfig = super.getPluginJobConf(); + + String type = originalConfig.getString(Key.SINK_DB_TYPE, Key.TYPE_DEFAULT_VALUE); + if (StringUtils.isBlank(type)) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.SINK_DB_TYPE + "] is not set."); + } + if (!Key.TYPE_SET.contains(type)) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.SINK_DB_TYPE + "] should be one of [" + + JSON.toJSONString(Key.TYPE_SET) + "]."); + } + + String address = originalConfig.getString(Key.ENDPOINT); + if (StringUtils.isBlank(address)) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.ENDPOINT + "] is not set."); + } + + // tagK / field could be empty + if ("TSDB".equals(type)) { + List columns = originalConfig.getList(Key.COLUMN, String.class); + if (columns == null || columns.isEmpty()) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.COLUMN + "] is not set."); + } + } else { + List columns = originalConfig.getList(Key.COLUMN, String.class); + if (columns == null || columns.isEmpty()) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.COLUMN + "] is not set."); + } + for (String specifyKey : Constant.MUST_CONTAINED_SPECIFY_KEYS) { + if (!columns.contains(specifyKey)) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.COLUMN + "] should contain " + + JSON.toJSONString(Constant.MUST_CONTAINED_SPECIFY_KEYS) + "."); + } + } + final List metrics = originalConfig.getList(Key.METRIC, String.class); + if (metrics == null || metrics.isEmpty()) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.METRIC + "] is not set."); + } + } + + Integer splitIntervalMs = originalConfig.getInt(Key.INTERVAL_DATE_TIME, + Key.INTERVAL_DATE_TIME_DEFAULT_VALUE); + if (splitIntervalMs <= 0) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.INTERVAL_DATE_TIME + "] should be great than zero."); + } + + SimpleDateFormat format = new SimpleDateFormat(Constant.DEFAULT_DATA_FORMAT); + String startTime = originalConfig.getString(Key.BEGIN_DATE_TIME); + Long startDate; + if (startTime == null || startTime.trim().length() == 0) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.BEGIN_DATE_TIME + "] is not set."); + } else { + try { + startDate = format.parse(startTime).getTime(); + } catch (ParseException e) { + throw DataXException.asDataXException(TSDBReaderErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.BEGIN_DATE_TIME + + "] needs to conform to the [" + Constant.DEFAULT_DATA_FORMAT + "] format."); + } + } + String endTime = originalConfig.getString(Key.END_DATE_TIME); + Long endDate; + if (endTime == null || endTime.trim().length() == 0) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.END_DATE_TIME + "] is not set."); + } else { + try { + endDate = format.parse(endTime).getTime(); + } catch (ParseException e) { + throw DataXException.asDataXException(TSDBReaderErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.END_DATE_TIME + + "] needs to conform to the [" + Constant.DEFAULT_DATA_FORMAT + "] format."); + } + } + if (startDate >= endDate) { + throw DataXException.asDataXException(TSDBReaderErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.BEGIN_DATE_TIME + + "] should be less than the parameter [" + Key.END_DATE_TIME + "]."); + } + } + + @Override + public void prepare() { + } + + @Override + public List split(int adviceNumber) { + List configurations = new ArrayList<>(); + + // get metrics + String type = originalConfig.getString(Key.SINK_DB_TYPE, Key.TYPE_DEFAULT_VALUE); + List columns4TSDB = null; + List columns4RDB = null; + List metrics = null; + if ("TSDB".equals(type)) { + columns4TSDB = originalConfig.getList(Key.COLUMN, String.class); + } else { + columns4RDB = originalConfig.getList(Key.COLUMN, String.class); + metrics = originalConfig.getList(Key.METRIC, String.class); + } + + // get time interval + Integer splitIntervalMs = originalConfig.getInt(Key.INTERVAL_DATE_TIME, + Key.INTERVAL_DATE_TIME_DEFAULT_VALUE); + + // get time range + SimpleDateFormat format = new SimpleDateFormat(Constant.DEFAULT_DATA_FORMAT); + long startTime; + try { + startTime = format.parse(originalConfig.getString(Key.BEGIN_DATE_TIME)).getTime(); + } catch (ParseException e) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.ILLEGAL_VALUE, "解析[" + Key.BEGIN_DATE_TIME + "]失败.", e); + } + long endTime; + try { + endTime = format.parse(originalConfig.getString(Key.END_DATE_TIME)).getTime(); + } catch (ParseException e) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.ILLEGAL_VALUE, "解析[" + Key.END_DATE_TIME + "]失败.", e); + } + if (TimeUtils.isSecond(startTime)) { + startTime *= 1000; + } + if (TimeUtils.isSecond(endTime)) { + endTime *= 1000; + } + DateTime startDateTime = new DateTime(TimeUtils.getTimeInHour(startTime)); + DateTime endDateTime = new DateTime(TimeUtils.getTimeInHour(endTime)); + + if ("TSDB".equals(type)) { + // split by metric + for (String column : columns4TSDB) { + // split by time in hour + while (startDateTime.isBefore(endDateTime)) { + Configuration clone = this.originalConfig.clone(); + clone.set(Key.COLUMN, Collections.singletonList(column)); + + clone.set(Key.BEGIN_DATE_TIME, startDateTime.getMillis()); + startDateTime = startDateTime.plusMillis(splitIntervalMs); + // Make sure the time interval is [start, end). + clone.set(Key.END_DATE_TIME, startDateTime.getMillis() - 1); + configurations.add(clone); + + LOG.info("Configuration: {}", JSON.toJSONString(clone)); + } + } + } else { + // split by metric + for (String metric : metrics) { + // split by time in hour + while (startDateTime.isBefore(endDateTime)) { + Configuration clone = this.originalConfig.clone(); + clone.set(Key.COLUMN, columns4RDB); + clone.set(Key.METRIC, Collections.singletonList(metric)); + + clone.set(Key.BEGIN_DATE_TIME, startDateTime.getMillis()); + startDateTime = startDateTime.plusMillis(splitIntervalMs); + // Make sure the time interval is [start, end). + clone.set(Key.END_DATE_TIME, startDateTime.getMillis() - 1); + configurations.add(clone); + + LOG.info("Configuration: {}", JSON.toJSONString(clone)); + } + } + } + return configurations; + } + + @Override + public void post() { + } + + @Override + public void destroy() { + } + } + + public static class Task extends Reader.Task { + + private static final Logger LOG = LoggerFactory.getLogger(Task.class); + + private String type; + private List columns4TSDB = null; + private List columns4RDB = null; + private List metrics = null; + private Map fields; + private Map tags; + private TSDBConnection conn; + private Long startTime; + private Long endTime; + + @Override + public void init() { + Configuration readerSliceConfig = super.getPluginJobConf(); + + LOG.info("getPluginJobConf: {}", JSON.toJSONString(readerSliceConfig)); + + this.type = readerSliceConfig.getString(Key.SINK_DB_TYPE); + if ("TSDB".equals(type)) { + columns4TSDB = readerSliceConfig.getList(Key.COLUMN, String.class); + } else { + columns4RDB = readerSliceConfig.getList(Key.COLUMN, String.class); + metrics = readerSliceConfig.getList(Key.METRIC, String.class); + } + this.fields = readerSliceConfig.getMap(Key.FIELD); + this.tags = readerSliceConfig.getMap(Key.TAG); + + String address = readerSliceConfig.getString(Key.ENDPOINT); + + conn = new TSDBConnection(address); + + this.startTime = readerSliceConfig.getLong(Key.BEGIN_DATE_TIME); + this.endTime = readerSliceConfig.getLong(Key.END_DATE_TIME); + } + + @Override + public void prepare() { + } + + @Override + @SuppressWarnings("unchecked") + public void startRead(RecordSender recordSender) { + try { + if ("TSDB".equals(type)) { + for (String metric : columns4TSDB) { + final Map tags = this.tags == null ? + null : (Map) this.tags.get(metric); + if (fields == null || !fields.containsKey(metric)) { + conn.sendDPs(metric, tags, this.startTime, this.endTime, recordSender); + } else { + conn.sendDPs(metric, (List) fields.get(metric), + tags, this.startTime, this.endTime, recordSender); + } + } + } else { + for (String metric : metrics) { + final Map tags = this.tags == null ? + null : (Map) this.tags.get(metric); + if (fields == null || !fields.containsKey(metric)) { + conn.sendRecords(metric, tags, startTime, endTime, columns4RDB, recordSender); + } else { + conn.sendRecords(metric, (List) fields.get(metric), + tags, startTime, endTime, columns4RDB, recordSender); + } + } + } + } catch (Exception e) { + throw DataXException.asDataXException( + TSDBReaderErrorCode.ILLEGAL_VALUE, "获取或发送数据点的过程中出错！", e); + } + } + + @Override + public void post() { + } + + @Override + public void destroy() { + } + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReaderErrorCode.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReaderErrorCode.java new file mode 100755 index 00000000..4ebdcca5 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReaderErrorCode.java @@ -0,0 +1,40 @@ +package com.alibaba.datax.plugin.reader.tsdbreader; + +import com.alibaba.datax.common.spi.ErrorCode; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：TSDB Reader Error Code + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public enum TSDBReaderErrorCode implements ErrorCode { + + REQUIRED_VALUE("TSDBReader-00", "缺失必要的值"), + ILLEGAL_VALUE("TSDBReader-01", "值非法"); + + private final String code; + private final String description; + + TSDBReaderErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/Connection4TSDB.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/Connection4TSDB.java new file mode 100644 index 00000000..500894bb --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/Connection4TSDB.java @@ -0,0 +1,88 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.conn; + +import com.alibaba.datax.common.plugin.RecordSender; + +import java.util.List; +import java.util.Map; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：Connection for TSDB-like databases + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public interface Connection4TSDB { + + /** + * Get the address of Database. + * + * @return host+ip + */ + String address(); + + /** + * Get the version of Database. + * + * @return version + */ + String version(); + + /** + * Get these configurations. + * + * @return configs + */ + String config(); + + /** + * Get the list of supported version. + * + * @return version list + */ + String[] getSupportVersionPrefix(); + + /** + * Send data points for TSDB with single field. + */ + void sendDPs(String metric, Map tags, Long start, Long end, RecordSender recordSender) throws Exception; + + /** + * Send data points for TSDB with multi fields. + */ + void sendDPs(String metric, List fields, Map tags, Long start, Long end, RecordSender recordSender) throws Exception; + + /** + * Send data points for RDB with single field. + */ + void sendRecords(String metric, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender) throws Exception; + + /** + * Send data points for RDB with multi fields. + */ + void sendRecords(String metric, List fields, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender) throws Exception; + + /** + * Put data point. + * + * @param dp data point + * @return whether the data point is written successfully + */ + boolean put(DataPoint4TSDB dp); + + /** + * Put data points. + * + * @param dps data points + * @return whether the data point is written successfully + */ + boolean put(List dps); + + /** + * Whether current version is supported. + * + * @return true: supported; false: not yet! + */ + boolean isSupported(); +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/DataPoint4MultiFieldsTSDB.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/DataPoint4MultiFieldsTSDB.java new file mode 100644 index 00000000..5b380c73 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/DataPoint4MultiFieldsTSDB.java @@ -0,0 +1,68 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.conn; + +import com.alibaba.fastjson.JSON; + +import java.util.Map; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：DataPoint for TSDB with Multi Fields + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public class DataPoint4MultiFieldsTSDB { + + private long timestamp; + private String metric; + private Map tags; + private Map fields; + + public DataPoint4MultiFieldsTSDB() { + } + + public DataPoint4MultiFieldsTSDB(long timestamp, String metric, Map tags, Map fields) { + this.timestamp = timestamp; + this.metric = metric; + this.tags = tags; + this.fields = fields; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public String getMetric() { + return metric; + } + + public void setMetric(String metric) { + this.metric = metric; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public Map getFields() { + return fields; + } + + public void setFields(Map fields) { + this.fields = fields; + } + + @Override + public String toString() { + return JSON.toJSONString(this); + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/DataPoint4TSDB.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/DataPoint4TSDB.java new file mode 100644 index 00000000..5c5c1349 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/DataPoint4TSDB.java @@ -0,0 +1,68 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.conn; + +import com.alibaba.fastjson.JSON; + +import java.util.Map; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：DataPoint for TSDB + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public class DataPoint4TSDB { + + private long timestamp; + private String metric; + private Map tags; + private Object value; + + public DataPoint4TSDB() { + } + + public DataPoint4TSDB(long timestamp, String metric, Map tags, Object value) { + this.timestamp = timestamp; + this.metric = metric; + this.tags = tags; + this.value = value; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public String getMetric() { + return metric; + } + + public void setMetric(String metric) { + this.metric = metric; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public Object getValue() { + return value; + } + + public void setValue(Object value) { + this.value = value; + } + + @Override + public String toString() { + return JSON.toJSONString(this); + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/MultiFieldQueryResult.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/MultiFieldQueryResult.java new file mode 100644 index 00000000..32c2a76e --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/MultiFieldQueryResult.java @@ -0,0 +1,64 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.conn; + +import java.util.List; +import java.util.Map; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：Multi Field Query Result + * + * @author Benedict Jin + * @since 2019-10-22 + */ +public class MultiFieldQueryResult { + + private String metric; + private Map tags; + private List aggregatedTags; + private List columns; + private List> values; + + public MultiFieldQueryResult() { + } + + public String getMetric() { + return metric; + } + + public void setMetric(String metric) { + this.metric = metric; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public List getAggregatedTags() { + return aggregatedTags; + } + + public void setAggregatedTags(List aggregatedTags) { + this.aggregatedTags = aggregatedTags; + } + + public List getColumns() { + return columns; + } + + public void setColumns(List columns) { + this.columns = columns; + } + + public List> getValues() { + return values; + } + + public void setValues(List> values) { + this.values = values; + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/QueryResult.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/QueryResult.java new file mode 100644 index 00000000..8f3e2b59 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/QueryResult.java @@ -0,0 +1,64 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.conn; + +import java.util.List; +import java.util.Map; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：Query Result + * + * @author Benedict Jin + * @since 2019-09-19 + */ +public class QueryResult { + + private String metricName; + private Map tags; + private List groupByTags; + private List aggregatedTags; + private Map dps; + + public QueryResult() { + } + + public String getMetricName() { + return metricName; + } + + public void setMetricName(String metricName) { + this.metricName = metricName; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public List getGroupByTags() { + return groupByTags; + } + + public void setGroupByTags(List groupByTags) { + this.groupByTags = groupByTags; + } + + public List getAggregatedTags() { + return aggregatedTags; + } + + public void setAggregatedTags(List aggregatedTags) { + this.aggregatedTags = aggregatedTags; + } + + public Map getDps() { + return dps; + } + + public void setDps(Map dps) { + this.dps = dps; + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnection.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnection.java new file mode 100644 index 00000000..5426ab49 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnection.java @@ -0,0 +1,94 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.conn; + +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.plugin.reader.tsdbreader.util.TSDBUtils; +import com.alibaba.fastjson.JSON; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：TSDB Connection + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public class TSDBConnection implements Connection4TSDB { + + private String address; + + public TSDBConnection(String address) { + this.address = address; + } + + @Override + public String address() { + return address; + } + + @Override + public String version() { + return TSDBUtils.version(address); + } + + @Override + public String config() { + return TSDBUtils.config(address); + } + + @Override + public String[] getSupportVersionPrefix() { + return new String[]{"2.4", "2.5"}; + } + + @Override + public void sendDPs(String metric, Map tags, Long start, Long end, RecordSender recordSender) throws Exception { + TSDBDump.dump4TSDB(this, metric, tags, start, end, recordSender); + } + + @Override + public void sendDPs(String metric, List fields, Map tags, Long start, Long end, RecordSender recordSender) throws Exception { + TSDBDump.dump4TSDB(this, metric, fields, tags, start, end, recordSender); + } + + @Override + public void sendRecords(String metric, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender) throws Exception { + TSDBDump.dump4RDB(this, metric, tags, start, end, columns4RDB, recordSender); + } + + @Override + public void sendRecords(String metric, List fields, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender) throws Exception { + TSDBDump.dump4RDB(this, metric, fields, tags, start, end, columns4RDB, recordSender); + } + + @Override + public boolean put(DataPoint4TSDB dp) { + return false; + } + + @Override + public boolean put(List dps) { + return false; + } + + @Override + public boolean isSupported() { + String versionJson = version(); + if (StringUtils.isBlank(versionJson)) { + throw new RuntimeException("Cannot get the version!"); + } + String version = JSON.parseObject(versionJson).getString("version"); + if (StringUtils.isBlank(version)) { + return false; + } + for (String prefix : getSupportVersionPrefix()) { + if (version.startsWith(prefix)) { + return true; + } + } + return false; + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBDump.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBDump.java new file mode 100644 index 00000000..8bae3a70 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBDump.java @@ -0,0 +1,318 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.conn; + +import com.alibaba.datax.common.element.*; +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.plugin.reader.tsdbreader.Constant; +import com.alibaba.datax.plugin.reader.tsdbreader.util.HttpUtils; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.parser.Feature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：TSDB Dump + * + * @author Benedict Jin + * @since 2019-10-21 + */ +final class TSDBDump { + + private static final Logger LOG = LoggerFactory.getLogger(TSDBDump.class); + + private static final String QUERY = "/api/query"; + private static final String QUERY_MULTI_FIELD = "/api/mquery"; + + static { + JSON.DEFAULT_PARSER_FEATURE &= ~Feature.UseBigDecimal.getMask(); + } + + private TSDBDump() { + } + + static void dump4TSDB(TSDBConnection conn, String metric, Map tags, + Long start, Long end, RecordSender sender) throws Exception { + LOG.info("conn address: {}, metric: {}, start: {}, end: {}", conn.address(), metric, start, end); + + String res = queryRange4SingleField(conn, metric, tags, start, end); + List dps = getDps4TSDB(metric, res); + if (dps == null || dps.isEmpty()) { + return; + } + sendTSDBDps(sender, dps); + } + + static void dump4TSDB(TSDBConnection conn, String metric, List fields, Map tags, + Long start, Long end, RecordSender sender) throws Exception { + LOG.info("conn address: {}, metric: {}, start: {}, end: {}", conn.address(), metric, start, end); + + String res = queryRange4MultiFields(conn, metric, fields, tags, start, end); + List dps = getDps4TSDB(metric, fields, res); + if (dps == null || dps.isEmpty()) { + return; + } + sendTSDBDps(sender, dps); + } + + static void dump4RDB(TSDBConnection conn, String metric, Map tags, + Long start, Long end, List columns4RDB, RecordSender sender) throws Exception { + LOG.info("conn address: {}, metric: {}, start: {}, end: {}", conn.address(), metric, start, end); + + String res = queryRange4SingleField(conn, metric, tags, start, end); + List dps = getDps4RDB(metric, res); + if (dps == null || dps.isEmpty()) { + return; + } + for (DataPoint4TSDB dp : dps) { + final Record record = sender.createRecord(); + final Map tagKV = dp.getTags(); + + for (String column : columns4RDB) { + if (Constant.METRIC_SPECIFY_KEY.equals(column)) { + record.addColumn(new StringColumn(dp.getMetric())); + } else if (Constant.TS_SPECIFY_KEY.equals(column)) { + record.addColumn(new LongColumn(dp.getTimestamp())); + } else if (Constant.VALUE_SPECIFY_KEY.equals(column)) { + record.addColumn(getColumn(dp.getValue())); + } else { + final Object tagk = tagKV.get(column); + if (tagk == null) { + continue; + } + record.addColumn(getColumn(tagk)); + } + } + sender.sendToWriter(record); + } + } + + static void dump4RDB(TSDBConnection conn, String metric, List fields, + Map tags, Long start, Long end, + List columns4RDB, RecordSender sender) throws Exception { + LOG.info("conn address: {}, metric: {}, start: {}, end: {}", conn.address(), metric, start, end); + + String res = queryRange4MultiFields(conn, metric, fields, tags, start, end); + List dps = getDps4RDB(metric, fields, res); + if (dps == null || dps.isEmpty()) { + return; + } + for (DataPoint4TSDB dp : dps) { + final Record record = sender.createRecord(); + final Map tagKV = dp.getTags(); + + for (String column : columns4RDB) { + if (Constant.METRIC_SPECIFY_KEY.equals(column)) { + record.addColumn(new StringColumn(dp.getMetric())); + } else if (Constant.TS_SPECIFY_KEY.equals(column)) { + record.addColumn(new LongColumn(dp.getTimestamp())); + } else { + final Object tagvOrField = tagKV.get(column); + if (tagvOrField == null) { + continue; + } + record.addColumn(getColumn(tagvOrField)); + } + } + sender.sendToWriter(record); + } + } + + private static Column getColumn(Object value) throws Exception { + Column valueColumn; + if (value instanceof Double) { + valueColumn = new DoubleColumn((Double) value); + } else if (value instanceof Long) { + valueColumn = new LongColumn((Long) value); + } else if (value instanceof String) { + valueColumn = new StringColumn((String) value); + } else { + throw new Exception(String.format("value 不支持类型: [%s]", value.getClass().getSimpleName())); + } + return valueColumn; + } + + private static String queryRange4SingleField(TSDBConnection conn, String metric, Map tags, + Long start, Long end) throws Exception { + String tagKV = getFilterByTags(tags); + String body = "{\n" + + " \"start\": " + start + ",\n" + + " \"end\": " + end + ",\n" + + " \"queries\": [\n" + + " {\n" + + " \"aggregator\": \"none\",\n" + + " \"metric\": \"" + metric + "\"\n" + + (tagKV == null ? "" : tagKV) + + " }\n" + + " ]\n" + + "}"; + return HttpUtils.post(conn.address() + QUERY, body); + } + + private static String queryRange4MultiFields(TSDBConnection conn, String metric, List fields, + Map tags, Long start, Long end) throws Exception { + // fields + StringBuilder fieldBuilder = new StringBuilder(); + fieldBuilder.append("\"fields\":["); + for (int i = 0; i < fields.size(); i++) { + fieldBuilder.append("{\"field\": \"").append(fields.get(i)).append("\",\"aggregator\": \"none\"}"); + if (i != fields.size() - 1) { + fieldBuilder.append(","); + } + } + fieldBuilder.append("]"); + // tagkv + String tagKV = getFilterByTags(tags); + String body = "{\n" + + " \"start\": " + start + ",\n" + + " \"end\": " + end + ",\n" + + " \"queries\": [\n" + + " {\n" + + " \"aggregator\": \"none\",\n" + + " \"metric\": \"" + metric + "\",\n" + + fieldBuilder.toString() + + (tagKV == null ? "" : tagKV) + + " }\n" + + " ]\n" + + "}"; + return HttpUtils.post(conn.address() + QUERY_MULTI_FIELD, body); + } + + private static String getFilterByTags(Map tags) { + if (tags != null && !tags.isEmpty()) { + // tagKV = ",\"tags:\":" + JSON.toJSONString(tags); + StringBuilder tagBuilder = new StringBuilder(); + tagBuilder.append(",\"filters\":["); + int count = 1; + final int size = tags.size(); + for (Map.Entry entry : tags.entrySet()) { + final String tagK = entry.getKey(); + final String tagV = entry.getValue(); + tagBuilder.append("{\"type\":\"literal_or\",\"tagk\":\"").append(tagK) + .append("\",\"filter\":\"").append(tagV).append("\",\"groupBy\":false}"); + if (count != size) { + tagBuilder.append(","); + } + count++; + } + tagBuilder.append("]"); + return tagBuilder.toString(); + } + return null; + } + + private static List getDps4TSDB(String metric, String dps) { + final List jsonArray = JSON.parseArray(dps, QueryResult.class); + if (jsonArray.size() == 0) { + return null; + } + List dpsArr = new LinkedList<>(); + for (QueryResult queryResult : jsonArray) { + final Map tags = queryResult.getTags(); + final Map points = queryResult.getDps(); + for (Map.Entry entry : points.entrySet()) { + final String ts = entry.getKey(); + final Object value = entry.getValue(); + + DataPoint4TSDB dp = new DataPoint4TSDB(); + dp.setMetric(metric); + dp.setTags(tags); + dp.setTimestamp(Long.parseLong(ts)); + dp.setValue(value); + dpsArr.add(dp.toString()); + } + } + return dpsArr; + } + + private static List getDps4TSDB(String metric, List fields, String dps) { + final List jsonArray = JSON.parseArray(dps, MultiFieldQueryResult.class); + if (jsonArray.size() == 0) { + return null; + } + List dpsArr = new LinkedList<>(); + for (MultiFieldQueryResult queryResult : jsonArray) { + final Map tags = queryResult.getTags(); + final List> values = queryResult.getValues(); + for (List value : values) { + final String ts = value.get(0).toString(); + Map fieldsAndValues = new HashMap<>(); + for (int i = 0; i < fields.size(); i++) { + fieldsAndValues.put(fields.get(i), value.get(i + 1)); + } + + final DataPoint4MultiFieldsTSDB dp = new DataPoint4MultiFieldsTSDB(); + dp.setMetric(metric); + dp.setTimestamp(Long.parseLong(ts)); + dp.setTags(tags); + dp.setFields(fieldsAndValues); + dpsArr.add(dp.toString()); + } + } + return dpsArr; + } + + private static List getDps4RDB(String metric, String dps) { + final List jsonArray = JSON.parseArray(dps, QueryResult.class); + if (jsonArray.size() == 0) { + return null; + } + List dpsArr = new LinkedList<>(); + for (QueryResult queryResult : jsonArray) { + final Map tags = queryResult.getTags(); + final Map points = queryResult.getDps(); + for (Map.Entry entry : points.entrySet()) { + final String ts = entry.getKey(); + final Object value = entry.getValue(); + + final DataPoint4TSDB dp = new DataPoint4TSDB(); + dp.setMetric(metric); + dp.setTags(tags); + dp.setTimestamp(Long.parseLong(ts)); + dp.setValue(value); + dpsArr.add(dp); + } + } + return dpsArr; + } + + private static List getDps4RDB(String metric, List fields, String dps) { + final List jsonArray = JSON.parseArray(dps, MultiFieldQueryResult.class); + if (jsonArray.size() == 0) { + return null; + } + List dpsArr = new LinkedList<>(); + for (MultiFieldQueryResult queryResult : jsonArray) { + final Map tags = queryResult.getTags(); + final List> values = queryResult.getValues(); + for (List value : values) { + final String ts = value.get(0).toString(); + Map tagsTmp = new HashMap<>(tags); + for (int i = 0; i < fields.size(); i++) { + tagsTmp.put(fields.get(i), value.get(i + 1)); + } + + final DataPoint4TSDB dp = new DataPoint4TSDB(); + dp.setMetric(metric); + dp.setTimestamp(Long.parseLong(ts)); + dp.setTags(tagsTmp); + dpsArr.add(dp); + } + } + return dpsArr; + } + + private static void sendTSDBDps(RecordSender sender, List dps) { + for (String dp : dps) { + StringColumn tsdbColumn = new StringColumn(dp); + Record record = sender.createRecord(); + record.addColumn(tsdbColumn); + sender.sendToWriter(record); + } + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtils.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtils.java new file mode 100644 index 00000000..3e0be854 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtils.java @@ -0,0 +1,67 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.util; + +import com.alibaba.fastjson.JSON; +import org.apache.http.client.fluent.Content; +import org.apache.http.client.fluent.Request; +import org.apache.http.entity.ContentType; + +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：HttpUtils + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public final class HttpUtils { + + public final static int CONNECT_TIMEOUT_DEFAULT_IN_MILL = (int) TimeUnit.SECONDS.toMillis(60); + public final static int SOCKET_TIMEOUT_DEFAULT_IN_MILL = (int) TimeUnit.SECONDS.toMillis(60); + + private HttpUtils() { + } + + public static String get(String url) throws Exception { + Content content = Request.Get(url) + .connectTimeout(CONNECT_TIMEOUT_DEFAULT_IN_MILL) + .socketTimeout(SOCKET_TIMEOUT_DEFAULT_IN_MILL) + .execute() + .returnContent(); + if (content == null) { + return null; + } + return content.asString(StandardCharsets.UTF_8); + } + + public static String post(String url, Map params) throws Exception { + return post(url, JSON.toJSONString(params), CONNECT_TIMEOUT_DEFAULT_IN_MILL, SOCKET_TIMEOUT_DEFAULT_IN_MILL); + } + + public static String post(String url, String params) throws Exception { + return post(url, params, CONNECT_TIMEOUT_DEFAULT_IN_MILL, SOCKET_TIMEOUT_DEFAULT_IN_MILL); + } + + public static String post(String url, Map params, + int connectTimeoutInMill, int socketTimeoutInMill) throws Exception { + return post(url, JSON.toJSONString(params), connectTimeoutInMill, socketTimeoutInMill); + } + + public static String post(String url, String params, + int connectTimeoutInMill, int socketTimeoutInMill) throws Exception { + Content content = Request.Post(url) + .connectTimeout(connectTimeoutInMill) + .socketTimeout(socketTimeoutInMill) + .addHeader("Content-Type", "application/json") + .bodyString(params, ContentType.APPLICATION_JSON) + .execute() + .returnContent(); + if (content == null) { + return null; + } + return content.asString(StandardCharsets.UTF_8); + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TSDBUtils.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TSDBUtils.java new file mode 100644 index 00000000..bb7b4b87 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TSDBUtils.java @@ -0,0 +1,68 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.util; + +import com.alibaba.datax.plugin.reader.tsdbreader.conn.DataPoint4TSDB; +import com.alibaba.fastjson.JSON; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：TSDB Utils + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public final class TSDBUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(TSDBUtils.class); + + private TSDBUtils() { + } + + public static String version(String address) { + String url = String.format("%s/api/version", address); + String rsp; + try { + rsp = HttpUtils.get(url); + } catch (Exception e) { + throw new RuntimeException(e); + } + return rsp; + } + + public static String config(String address) { + String url = String.format("%s/api/config", address); + String rsp; + try { + rsp = HttpUtils.get(url); + } catch (Exception e) { + throw new RuntimeException(e); + } + return rsp; + } + + public static boolean put(String address, List dps) { + return put(address, JSON.toJSON(dps)); + } + + public static boolean put(String address, DataPoint4TSDB dp) { + return put(address, JSON.toJSON(dp)); + } + + private static boolean put(String address, Object o) { + String url = String.format("%s/api/put", address); + String rsp; + try { + rsp = HttpUtils.post(url, o.toString()); + // If successful, the returned content should be null. + assert rsp == null; + } catch (Exception e) { + LOGGER.error("Address: {}, DataPoints: {}", url, o); + throw new RuntimeException(e); + } + return true; + } +} diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TimeUtils.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TimeUtils.java new file mode 100644 index 00000000..c1243164 --- /dev/null +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TimeUtils.java @@ -0,0 +1,38 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.util; + +import java.util.concurrent.TimeUnit; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：TimeUtils + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public final class TimeUtils { + + private TimeUtils() { + } + + private static final long SECOND_MASK = 0xFFFFFFFF00000000L; + private static final long HOUR_IN_MILL = TimeUnit.HOURS.toMillis(1); + + /** + * Weather the timestamp is second. + * + * @param ts timestamp + */ + public static boolean isSecond(long ts) { + return (ts & SECOND_MASK) == 0; + } + + /** + * Get the hour. + * + * @param ms time in millisecond + */ + public static long getTimeInHour(long ms) { + return ms - ms % HOUR_IN_MILL; + } +} diff --git a/tsdbreader/src/main/resources/plugin.json b/tsdbreader/src/main/resources/plugin.json new file mode 100755 index 00000000..f2dbb1f0 --- /dev/null +++ b/tsdbreader/src/main/resources/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "tsdbreader", + "class": "com.alibaba.datax.plugin.reader.tsdbreader.TSDBReader", + "description": { + "useScene": "从 TSDB 中摄取数据点", + "mechanism": "通过 /api/query 接口查询出符合条件的数据点", + "warn": "指定起止时间会自动忽略分钟和秒，转为整点时刻，例如 2019-4-18 的 [3:35, 4:55) 会被转为 [3:00, 4:00)" + }, + "developer": "Benedict Jin" +} diff --git a/tsdbreader/src/main/resources/plugin_job_template.json b/tsdbreader/src/main/resources/plugin_job_template.json new file mode 100644 index 00000000..8570abb9 --- /dev/null +++ b/tsdbreader/src/main/resources/plugin_job_template.json @@ -0,0 +1,29 @@ +{ + "name": "tsdbreader", + "parameter": { + "sinkDbType": "RDB", + "endpoint": "http://localhost:8242", + "column": [ + "__metric__", + "__ts__", + "app", + "cluster", + "group", + "ip", + "zone", + "__value__" + ], + "metric": [ + "m" + ], + "tag": { + "m": { + "app": "a1", + "cluster": "c1" + } + }, + "splitIntervalMs": 60000, + "beginDateTime": "2019-01-01 00:00:00", + "endDateTime": "2019-01-01 01:00:00" + } +} diff --git a/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnectionTest.java b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnectionTest.java new file mode 100644 index 00000000..e4544088 --- /dev/null +++ b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnectionTest.java @@ -0,0 +1,30 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.conn; + +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：TSDB Connection4TSDB Test + * + * @author Benedict Jin + * @since 2019-10-21 + */ +@Ignore +public class TSDBConnectionTest { + + private static final String TSDB_ADDRESS = "http://localhost:8242"; + + @Test + public void testVersion() { + String version = new TSDBConnection(TSDB_ADDRESS).version(); + Assert.assertNotNull(version); + } + + @Test + public void testIsSupported() { + Assert.assertTrue(new TSDBConnection(TSDB_ADDRESS).isSupported()); + } +} diff --git a/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/Const.java b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/Const.java new file mode 100644 index 00000000..43e04b5c --- /dev/null +++ b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/Const.java @@ -0,0 +1,17 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.util; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：Const + * + * @author Benedict Jin + * @since 2019-10-21 + */ +final class Const { + + private Const() { + } + + static final String TSDB_ADDRESS = "http://localhost:8242"; +} diff --git a/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtilsTest.java b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtilsTest.java new file mode 100644 index 00000000..12a2660a --- /dev/null +++ b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtilsTest.java @@ -0,0 +1,39 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.util; + +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：HttpUtils Test + * + * @author Benedict Jin + * @since 2019-10-21 + */ +@Ignore +public class HttpUtilsTest { + + @Test + public void testSimpleCase() throws Exception { + String url = "https://httpbin.org/post"; + Map params = new HashMap<>(); + params.put("foo", "bar"); + + String rsp = HttpUtils.post(url, params); + System.out.println(rsp); + Assert.assertNotNull(rsp); + } + + @Test + public void testGet() throws Exception { + String url = String.format("%s/api/version", Const.TSDB_ADDRESS); + String rsp = HttpUtils.get(url); + System.out.println(rsp); + Assert.assertNotNull(rsp); + } +} diff --git a/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TimeUtilsTest.java b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TimeUtilsTest.java new file mode 100644 index 00000000..cef8efae --- /dev/null +++ b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TimeUtilsTest.java @@ -0,0 +1,33 @@ +package com.alibaba.datax.plugin.reader.tsdbreader.util; + +import org.junit.Assert; +import org.junit.Test; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * Copyright @ 2019 alibaba.com + * All right reserved. + * Function：com.alibaba.datax.common.util + * + * @author Benedict Jin + * @since 2019-10-21 + */ +public class TimeUtilsTest { + + @Test + public void testIsSecond() { + Assert.assertFalse(TimeUtils.isSecond(System.currentTimeMillis())); + Assert.assertTrue(TimeUtils.isSecond(System.currentTimeMillis() / 1000)); + } + + @Test + public void testGetTimeInHour() throws ParseException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Date date = sdf.parse("2019-04-18 15:32:33"); + long timeInHour = TimeUtils.getTimeInHour(date.getTime()); + Assert.assertEquals("2019-04-18 15:00:00", sdf.format(timeInHour)); + } +} From c2effde23536423a8691ff2fd75039855e172160 Mon Sep 17 00:00:00 2001 From: asdf2014 Date: Fri, 8 Nov 2019 20:14:55 +0800 Subject: [PATCH 4/4] Add TSDB Reader into README doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e806ba2b..13bd79ba 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N | | HDFS | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/hdfsreader/doc/hdfsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md)| | | Elasticsearch | | √ |[写](https://github.com/alibaba/DataX/blob/master/elasticsearchwriter/doc/elasticsearchwriter.md)| | 时间序列数据库 | OpenTSDB | √ | |[读](https://github.com/alibaba/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md)| -| | TSDB | | √ |[写](https://github.com/alibaba/DataX/blob/master/tsdbwriter/doc/tsdbhttpwriter.md)| +| | TSDB | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/tsdbreader/doc/tsdbreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/tsdbwriter/doc/tsdbhttpwriter.md)| # 我要开发新的插件 请点击：[DataX插件开发宝典](https://github.com/alibaba/DataX/blob/master/dataxPluginDev.md)