diff --git a/otsreader/doc/otsreader.md b/otsreader/doc/otsreader.md
index 1297dbd6..77b4edfe 100644
--- a/otsreader/doc/otsreader.md
+++ b/otsreader/doc/otsreader.md
@@ -13,7 +13,7 @@ OTSReader插件实现了从OTS读取数据，并可以通过用户指定抽取
* 范围抽取
* 指定分片抽取
-OTS是构建在阿里云飞天分布式系统之上的 NoSQL数据库服务，提供海量结构化数据的存储和实时访问。OTS 以实例和表的形式组织数据，通过数据分片和负载均衡技术，实现规模上的无缝扩展。
+本版本的OTSReader新增了支持多版本数据的读取功能，同时兼容旧版本的配置文件
## 2 实现原理
@@ -25,201 +25,425 @@ OTSReader会根据OTS的表范围，按照Datax并发的数目N，将范围等
### 3.1 配置样例
-* 配置一个从OTS全表同步抽取数据到本地的作业:
+#### 3.1.1
+* 配置一个从OTS表读取单版本数据的reader:
```
{
- "job": {
- "setting": {
- },
- "content": [
- {
- "reader": {
- "name": "otsreader",
- "parameter": {
- /* ----------- 必填 --------------*/
- "endpoint":"",
- "accessId":"",
- "accessKey":"",
- "instanceName":"",
-
- // 导出数据表的表名
- "table":"",
-
- // 需要导出的列名，支持重复列和常量列，区分大小写
- // 常量列：类型支持STRING，INT，DOUBLE，BOOL和BINARY
- // 备注：BINARY需要通过Base64转换为对应的字符串传入插件
- "column":[
- {"name":"col1"}, // 普通列
- {"name":"col2"}, // 普通列
- {"name":"col3"}, // 普通列
- {"type":"STRING", "value" : "bazhen"}, // 常量列(字符串)
- {"type":"INT", "value" : ""}, // 常量列(整形)
- {"type":"DOUBLE", "value" : ""}, // 常量列(浮点)
- {"type":"BOOL", "value" : ""}, // 常量列(布尔)
- {"type":"BINARY", "value" : "Base64(bin)"} // 常量列(二进制),使用Base64编码完成
- ],
- "range":{
- // 导出数据的起始范围
- // 支持INF_MIN, INF_MAX, STRING, INT
- "begin":[
- {"type":"INF_MIN"},
- ],
- // 导出数据的结束范围
- // 支持INF_MIN, INF_MAX, STRING, INT
- "end":[
- {"type":"INF_MAX"},
- ]
- }
- }
- },
- "writer": {}
- }
- ]
- }
-}
-```
-
-* 配置一个定义抽取范围的OTSReader：
-
-```
-{
- "job": {
- "setting": {
- "speed": {
- "byte":10485760
+ "job": {
+ "setting": {
+ "speed": {
+ //设置传输速度，单位为byte/s，DataX运行会尽可能达到该速度但是不超过它.
+ "byte": 1048576
+ }
+ //出错限制
+ "errorLimit": {
+ //出错的record条数上限，当大于该值即报错。
+ "record": 0,
+ //出错的record百分比上限 1.0表示100%，0.02表示2%
+ "percentage": 0.02
+ }
+ },
+ "content": [
+ {
+ "reader": {
+ "name": "otsreader-internal",
+ "parameter": {
+ "endpoint":"",
+ "accessId":"",
+ "accessKey":"",
+ "instanceName":"",
+ "table": "",
+ //version定义了是否使用新版本插件 可选值：false || true
+ "newVersion":"false",
+ //mode定义了读取数据的格式（普通数据/多版本数据），可选值：normal || multiversion
+ "mode": "normal",
+
+ // 导出的范围,读取的范围是[begin,end)，左闭右开的区间
+ // begin小于end，表示正序读取数据
+ // begin大于end，表示反序读取数据
+ // begin和end不能相等
+ // type支持的类型有如下几类：
+ // string、int、binary
+ // binary输入的方式采用二进制的Base64字符串形式传入
+ // INF_MIN 表示无限小
+ // INF_MAX 表示无限大
+ "range":{
+ // 可选，默认表示从无限小开始读取
+ // 这个值的输入可以填写空数组，或者PK前缀，亦或者完整的PK，在正序读取数据时，默认填充PK后缀为INF_MIN，反序为INF_MAX
+ // 例子：
+ // 如果用户的表有2个PK，类型分别为string、int，那么如下3种输入都是合法，如：
+ // 1. [] --> 表示从表的开始位置读取
+ // 2. [{"type":"string", "value":"a"}] --> 表示从[{"type":"string", "value":"a"},{"type":"INF_MIN"}]
+ // 3. [{"type":"string", "value":"a"},{"type":"INF_MIN"}]
+ //
+ // binary类型的PK列比较特殊，因为Json不支持直接输入二进制数，所以系统定义:用户如果要传入
+ // 二进制，必须使用(Java)Base64.encodeBase64String方法，将二进制转换为一个可视化的字符串，然后将这个字符串填入value中
+ // 例子(Java)：
+ // byte[] bytes = "hello".getBytes(); # 构造一个二进制数据，这里使用字符串hello的byte值
+ // String inputValue = Base64.encodeBase64String(bytes) # 调用Base64方法，将二进制转换为可视化的字符串
+ // 上面的代码执行之后，可以获得inputValue为"aGVsbG8="
+ // 最终写入配置：{"type":"binary","value" : "aGVsbG8="}
+
+ "begin":[{"type":"string", "value":"a"},{"type":"INF_MIN"}],
+
+ // 默认表示读取到无限大结束
+ // 这个值得输入可以填写空数组，或者PK前缀，亦或者完整的PK，在正序读取数据时，默认填充PK后缀为INF_MAX，反序为INF_MIN
+ // 可选
+ "end":[{"type":"string", "value":"a"},{"type":"INF_MAX"}],
+
+ // 当前用户数据较多时，需要开启并发导出，Split可以将当前范围的的数据按照切分点切分为多个并发任务
+ // 可选
+ // 1. split中的输入值只能PK的第一列（分片建），且值的类型必须和PartitionKey一致
+ // 2. 值的范围必须在begin和end之间
+ // 3. split内部的值必须根据begin和end的正反序关系而递增或者递减
+ "split":[{"type":"string", "value":"b"}, {"type":"string", "value":"c"}]
},
- "errorLimit":0.0
+
+
+ // 指定要导出的列，支持普通列和常量列
+ // 格式
+ // 普通列格式：{"name":"{your column name}"}
+ // 常量列格式：{"type":"", "value":""} , type支持string、int、binary、bool、double
+ // binary类型需要使用base64转换成对应的字符串传入
+ // 注意：
+ // 1. PK列也是需要用户在下面单独指定
+ "column": [
+ {"name":"pk1"}, // 普通列，下同
+ {"name":"pk2"},
+ {"name":"attr1"},
+ {"type":"string","value" : ""} // 指定常量列，下同
+ {"type":"int","value" : ""}
+ {"type":"double","value" : ""}
+ // binary类型的常量列比较特殊，因为Json不支持直接输入二进制数，所以系统定义:用户如果要传入
+ // 二进制，必须使用(Java)Base64.encodeBase64String方法，将二进制转换为一个可视化的字符串，然后将这个字符串填入value中
+ // 例子(Java)：
+ // byte[] bytes = "hello".getBytes(); # 构造一个二进制数据，这里使用字符串hello的byte值
+ // String inputValue = Base64.encodeBase64String(bytes) # 调用Base64方法，将二进制转换为可视化的字符串
+ // 上面的代码执行之后，可以获得inputValue为"aGVsbG8="
+ // 最终写入配置：{"type":"binary","value" : "aGVsbG8="}
+
+ {"type":"binary","value" : "aGVsbG8="}
+ ],
+ }
},
- "content": [
- {
- "reader": {
- "name": "otsreader",
- "parameter": {
- "endpoint":"",
- "accessId":"",
- "accessKey":"",
- "instanceName":"",
-
- // 导出数据表的表名
- "table":"",
-
- // 需要导出的列名，支持重复类和常量列，区分大小写
- // 常量列：类型支持STRING，INT，DOUBLE，BOOL和BINARY
- // 备注：BINARY需要通过Base64转换为对应的字符串传入插件
- "column":[
- {"name":"col1"}, // 普通列
- {"name":"col2"}, // 普通列
- {"name":"col3"}, // 普通列
- {"type":"STRING","value" : ""}, // 常量列(字符串)
- {"type":"INT","value" : ""}, // 常量列(整形)
- {"type":"DOUBLE","value" : ""}, // 常量列(浮点)
- {"type":"BOOL","value" : ""}, // 常量列(布尔)
- {"type":"BINARY","value" : "Base64(bin)"} // 常量列(二进制)
- ],
- "range":{
- // 导出数据的起始范围
- // 支持INF_MIN, INF_MAX, STRING, INT
- "begin":[
- {"type":"INF_MIN"},
- {"type":"INF_MAX"},
- {"type":"STRING", "value":"hello"},
- {"type":"INT", "value":"2999"},
- ],
- // 导出数据的结束范围
- // 支持INF_MIN, INF_MAX, STRING, INT
- "end":[
- {"type":"INF_MAX"},
- {"type":"INF_MIN"},
- {"type":"STRING", "value":"hello"},
- {"type":"INT", "value":"2999"},
- ]
- }
- }
- },
- "writer": {}
- }
- ]
- }
+ "writer": {
+ //writer类型
+ "name": "streamwriter",
+ //是否打印内容
+ "parameter": {
+ "print": true
+ }
+ }
+ }
+ ]
+ }
}
```
+#### 3.1.2
+* 配置一个从OTS表读取多版本数据的reader(仅在newVersion == true时支持):
+
+```
+{
+ "job": {
+ "setting": {
+ "speed": {
+ //设置传输速度，单位为byte/s，DataX运行会尽可能达到该速度但是不超过它.
+ "byte": 1048576
+ }
+ //出错限制
+ "errorLimit": {
+ //出错的record条数上限，当大于该值即报错。
+ "record": 0,
+ //出错的record百分比上限 1.0表示100%，0.02表示2%
+ "percentage": 0.02
+ }
+ },
+ "content": [
+ {
+ "reader": {
+ "name": "otsreader-internal",
+ "parameter": {
+ "endpoint":"",
+ "accessId":"",
+ "accessKey":"",
+ "instanceName":"",
+ "table": "",
+ //version定义了是否使用新版本插件 可选值：false || true
+ "newVersion":"true",
+ //mode定义了读取数据的格式（普通数据/多版本数据），可选值：normal || multiversion
+ "mode": "multiversion",
+
+ // 导出的范围，,读取的范围是[begin,end)，左闭右开的区间
+ // begin小于end，表示正序读取数据
+ // begin大于end，表示反序读取数据
+ // begin和end不能相等
+ // type支持的类型有如下几类：
+ // string、int、binary
+ // binary输入的方式采用二进制的Base64字符串形式传入
+ // INF_MIN 表示无限小
+ // INF_MAX 表示无限大
+ "range":{
+ // 可选，默认表示从无限小开始读取
+ // 这个值的输入可以填写空数组，或者PK前缀，亦或者完整的PK，在正序读取数据时，默认填充PK后缀为INF_MIN，反序为INF_MAX
+ // 例子：
+ // 如果用户的表有2个PK，类型分别为string、int，那么如下3种输入都是合法，如：
+ // 1. [] --> 表示从表的开始位置读取
+ // 2. [{"type":"string", "value":"a"}] --> 表示从[{"type":"string", "value":"a"},{"type":"INF_MIN"}]
+ // 3. [{"type":"string", "value":"a"},{"type":"INF_MIN"}]
+ //
+ // binary类型的PK列比较特殊，因为Json不支持直接输入二进制数，所以系统定义:用户如果要传入
+ // 二进制，必须使用(Java)Base64.encodeBase64String方法，将二进制转换为一个可视化的字符串，然后将这个字符串填入value中
+ // 例子(Java)：
+ // byte[] bytes = "hello".getBytes(); # 构造一个二进制数据，这里使用字符串hello的byte值
+ // String inputValue = Base64.encodeBase64String(bytes) # 调用Base64方法，将二进制转换为可视化的字符串
+ // 上面的代码执行之后，可以获得inputValue为"aGVsbG8="
+ // 最终写入配置：{"type":"binary","value" : "aGVsbG8="}
+
+ "begin":[{"type":"string", "value":"a"},{"type":"INF_MIN"}],
+
+ // 默认表示读取到无限大结束
+ // 这个值得输入可以填写空数组，或者PK前缀，亦或者完整的PK，在正序读取数据时，默认填充PK后缀为INF_MAX，反序为INF_MIN
+ // 可选
+ "end":[{"type":"string", "value":"g"},{"type":"INF_MAX"}],
+
+ // 当前用户数据较多时，需要开启并发导出，Split可以将当前范围的的数据按照切分点切分为多个并发任务
+ // 可选
+ // 1. split中的输入值只能PK的第一列（分片建），且值的类型必须和PartitionKey一致
+ // 2. 值的范围必须在begin和end之间
+ // 3. split内部的值必须根据begin和end的正反序关系而递增或者递减
+ "split":[{"type":"string", "value":"b"}, {"type":"string", "value":"c"}]
+ },
+
+ // 指定要导出的列，在多版本模式下只支持普通列
+ // 格式：
+ // 普通列格式：{"name":"{your column name}"}
+ // 可选，默认导出所有列的所有版本
+ // 注意：
+ // 1.在多版本模式下，不支持常量列
+ // 2.PK列不能指定，导出4元组中默认包括完整的PK
+ // 3.不能重复指定列
+ "column": [
+ {"name":"attr1"}
+ ],
+
+ // 请求数据的Time Range,读取的范围是[begin,end)，左闭右开的区间
+ // 可选，默认读取全部版本
+ // 注意：begin必须小于end
+ "timeRange":{
+ // 可选，默认为0
+ // 取值范围是0~LONG_MAX
+ "begin":1400000000,
+ // 可选，默认为Long Max(9223372036854775807L)
+ // 取值范围是0~LONG_MAX
+ "end" :1600000000
+ },
+
+ // 请求的指定Version
+ // 可选，默认读取所有版本
+ // 取值范围是1~INT32_MAX
+ "maxVersion":10,
+ }
+ },
+ "writer": {
+ //writer类型
+ "name": "streamwriter",
+ //是否打印内容
+ "parameter": {
+ "print": true
+ }
+ }
+ }
+ ]
+ }
+}
+```
+#### 3.1.3
+* 配置一个从OTS **时序表**读取数据的reader(仅在newVersion == true时支持):
+```json
+{
+ "job": {
+ "setting": {
+ "speed": {
+ // 读取时序数据的通道数
+ "channel": 5
+ }
+ },
+ "content": [
+ {
+ "reader": {
+ "name": "otsreader",
+ "parameter": {
+ "endpoint": "",
+ "accessId": "",
+ "accessKey": "",
+ "instanceName": "",
+ "table": "",
+ // 读时序数据mode必须为normal
+ "mode": "normal",
+ // 读时序数据newVersion必须为true
+ "newVersion": "true",
+ // 配置该表为时序表
+ "isTimeseriesTable":"true",
+ // 配置需要读取时间线的measurementName字段，非必需
+ // 为空则读取全表数据
+ "measurementName":"measurement_5",
+ // column是一个数组，每个元素表示一列
+ // 对于常量列，需要配置以下字段:
+ // 1. type : 字段值类型，必需
+ // 支持类型 : string, int, double, bool, binary
+ // 2. value : 字段值，必需
+ //
+ // 对于普通列，需要配置以下字段:
+ // 1. name : 列名,必需
+ // 时间线的'度量名称'使用_m_name标识，数据类型为String
+ // 时间线的'数据源'使用_data_source标识，数据类型为String
+ // 时间线的'标签'使用_tags标识，数据类型为String
+ // 时间线的'时间戳'使用_time标识，数据类型为Long
+ // 2. is_timeseries_tag : 是否为tags字段内部的键值，非必需，默认为false。
+ // 3. type : 字段值类型，非必需，默认为string。
+ // 支持类型 : string, int, double, bool, binary
+ "column": [
+ {
+ "name": "_m_name"
+ },
+ {
+ "name": "tagA",
+ "is_timeseries_tag":"true"
+ },
+ {
+ "name": "double_0",
+ "type":"DOUBLE"
+ },
+ {
+ "name": "string_0",
+ "type":"STRING"
+ },
+ {
+ "name": "long_0",
+ "type":"int"
+ },
+ {
+ "name": "binary_0",
+ "type":"BINARY"
+ },
+ {
+ "name": "bool_0",
+ "type":"BOOL"
+ },
+ {
+ "type":"STRING",
+ "value":"testString"
+ }
+ ]
+ }
+ },
+ "writer": {
+
+ }
+ }
+ ]
+ }
+}
+
+```
### 3.2 参数说明
* **endpoint**
- * 描述：OTS Server的EndPoint地址，例如http://bazhen.cn−hangzhou.ots.aliyuncs.com。
+ * 描述：OTS Server的EndPoint地址，例如http://bazhen.cn−hangzhou.ots.aliyuncs.com。
- * 必选：是
+ * 必选：是
- * 默认值：无
+ * 默认值：无
* **accessId**
- * 描述：OTS的accessId
+ * 描述：OTS的accessId
- * 必选：是
+ * 必选：是
- * 默认值：无
+ * 默认值：无
* **accessKey**
- * 描述：OTS的accessKey
+ * 描述：OTS的accessKey
- * 必选：是
+ * 必选：是
- * 默认值：无
+ * 默认值：无
* **instanceName**
- * 描述：OTS的实例名称，实例是用户使用和管理 OTS 服务的实体，用户在开通 OTS 服务之后，需要通过管理控制台来创建实例，然后在实例内进行表的创建和管理。实例是 OTS 资源管理的基础单元，OTS 对应用程序的访问控制和资源计量都在实例级别完成。
+ * 描述：OTS的实例名称，实例是用户使用和管理 OTS 服务的实体，用户在开通 OTS 服务之后，需要通过管理控制台来创建实例，然后在实例内进行表的创建和管理。实例是 OTS 资源管理的基础单元，OTS 对应用程序的访问控制和资源计量都在实例级别完成。
- * 必选：是
+ * 必选：是
- * 默认值：无
+ * 默认值：无
* **table**
- * 描述：所选取的需要抽取的表名称，这里有且只能填写一张表。在OTS不存在多表同步的需求。
+ * 描述：所选取的需要抽取的表名称，这里有且只能填写一张表。在OTS不存在多表同步的需求。
- * 必选：是
+ * 必选：是
- * 默认值：无
+ * 默认值：无
+
+* **newVersion**
+
+ * 描述：version定义了使用的ots SDK版本。
+ * true，新版本插件，使用com.alicloud.openservices.tablestore的依赖（推荐）
+ * false，旧版本插件，使用com.aliyun.openservices.ots的依赖，**不支持多版本数据的读取**
+
+ * 必选：否
+
+ * 默认值：false
+
+* **mode**
+
+ * 描述：读取为多版本格式的数据，目前有两种模式。
+ * normal，对应普通的数据
+ * multiVersion，写入数据为多版本格式的数据，多版本模式下，配置参数有所不同，详见3.1.2
+
+ * 必选：否
+
+ * 默认值：normal
* **column**
- * 描述：所配置的表中需要同步的列名集合，使用JSON的数组描述字段信息。由于OTS本身是NoSQL系统，在OTSReader抽取数据过程中，必须指定相应地字段名称。
+ * 描述：所配置的表中需要同步的列名集合，使用JSON的数组描述字段信息。由于OTS本身是NoSQL系统，在OTSReader抽取数据过程中，必须指定相应地字段名称。
- 支持普通的列读取，例如: {"name":"col1"}
+ 支持普通的列读取，例如: {"name":"col1"}
- 支持部分列读取，如用户不配置该列，则OTSReader不予读取。
+ 支持部分列读取，如用户不配置该列，则OTSReader不予读取。
- 支持常量列读取，例如: {"type":"STRING", "value" : "DataX"}。使用type描述常量类型，目前支持STRING、INT、DOUBLE、BOOL、BINARY(用户使用Base64编码填写)、INF_MIN(OTS的系统限定最小值，使用该值用户不能填写value属性，否则报错)、INF_MAX(OTS的系统限定最大值，使用该值用户不能填写value属性，否则报错)。
+ 支持常量列读取，例如: {"type":"STRING", "value" : "DataX"}。使用type描述常量类型，目前支持STRING、INT、DOUBLE、BOOL、BINARY(用户使用Base64编码填写)、INF_MIN(OTS的系统限定最小值，使用该值用户不能填写value属性，否则报错)、INF_MAX(OTS的系统限定最大值，使用该值用户不能填写value属性，否则报错)。
- 不支持函数或者自定义表达式，由于OTS本身不提供类似SQL的函数或者表达式功能，OTSReader也不能提供函数或表达式列功能。
+ 不支持函数或者自定义表达式，由于OTS本身不提供类似SQL的函数或者表达式功能，OTSReader也不能提供函数或表达式列功能。
- * 必选：是
+ * 必选：是
- * 默认值：无
+ * 默认值：无
* **begin/end**
- * 描述：该配置项必须配对使用，用于支持OTS表范围抽取。begin/end中描述的是OTS **PrimaryKey**的区间分布状态，而且必须保证区间覆盖到所有的PrimaryKey，**需要指定该表下所有的PrimaryKey范围，不能遗漏任意一个PrimaryKey**，对于无限大小的区间，可以使用{"type":"INF_MIN"}，{"type":"INF_MAX"}指代。例如对一张主键为 [DeviceID, SellerID]的OTS进行抽取任务，begin/end可以配置为:
+ * 描述：该配置项必须配对使用，用于支持OTS表范围抽取。begin/end中描述的是OTS **PrimaryKey**的区间分布状态，而且必须保证区间覆盖到所有的PrimaryKey，**需要指定该表下所有的PrimaryKey范围，不能遗漏任意一个PrimaryKey**，对于无限大小的区间，可以使用{"type":"INF_MIN"}，{"type":"INF_MAX"}指代。例如对一张主键为 [DeviceID, SellerID]的OTS进行抽取任务，begin/end可以配置为:
- ```json
- "range": {
- "begin": {
- {"type":"INF_MIN"}, //指定deviceID最小值
- {"type":"INT", "value":"0"} //指定deviceID最小值
- },
- "end": {
- {"type":"INF_MAX"}, //指定deviceID抽取最大值
- {"type":"INT", "value":"9999"} //指定deviceID抽取最大值
- }
- }
- ```
+ ```json
+ "range": {
+ "begin": {
+ {"type":"INF_MIN"}, //指定deviceID最小值
+ {"type":"INT", "value":"0"} //指定deviceID最小值
+ },
+ "end": {
+ {"type":"INF_MAX"}, //指定deviceID抽取最大值
+ {"type":"INT", "value":"9999"} //指定deviceID抽取最大值
+ }
+ }
+ ```
如果要对上述表抽取全表，可以使用如下配置：
@@ -237,42 +461,42 @@ OTSReader会根据OTS的表范围，按照Datax并发的数目N，将范围等
}
```
- * 必选：是
+ * 必选：否
- * 默认值：空
+ * 默认值：读取全部值
* **split**
- * 描述：该配置项属于高级配置项，是用户自己定义切分配置信息，普通情况下不建议用户使用。适用场景通常在OTS数据存储发生热点，使用OTSReader自动切分的策略不能生效情况下，使用用户自定义的切分规则。split指定是的在Begin、End区间内的切分点，且只能是partitionKey的切分点信息，即在split仅配置partitionKey，而不需要指定全部的PrimaryKey。
+ * 描述：该配置项属于高级配置项，是用户自己定义切分配置信息，普通情况下不建议用户使用。适用场景通常在OTS数据存储发生热点，使用OTSReader自动切分的策略不能生效情况下，使用用户自定义的切分规则。split指定是的在Begin、End区间内的切分点，且只能是partitionKey的切分点信息，即在split仅配置partitionKey，而不需要指定全部的PrimaryKey。
- 例如对一张主键为 [DeviceID, SellerID]的OTS进行抽取任务，可以配置为:
+ 例如对一张主键为 [DeviceID, SellerID]的OTS进行抽取任务，可以配置为:
- ```json
- "range": {
- "begin": {
- {"type":"INF_MIN"}, //指定deviceID最小值
- {"type":"INF_MIN"} //指定deviceID最小值
- },
- "end": {
- {"type":"INF_MAX"}, //指定deviceID抽取最大值
- {"type":"INF_MAX"} //指定deviceID抽取最大值
- }，
- // 用户指定的切分点，如果指定了切分点，Job将按照begin、end和split进行Task的切分，
- // 切分的列只能是Partition Key（ParimaryKey的第一列）
- // 支持INF_MIN, INF_MAX, STRING, INT
- "split":[
- {"type":"STRING", "value":"1"},
- {"type":"STRING", "value":"2"},
- {"type":"STRING", "value":"3"},
- {"type":"STRING", "value":"4"},
- {"type":"STRING", "value":"5"}
- ]
- }
- ```
+ ```json
+ "range": {
+ "begin": {
+ {"type":"INF_MIN"}, //指定deviceID最小值
+ {"type":"INF_MIN"} //指定deviceID最小值
+ },
+ "end": {
+ {"type":"INF_MAX"}, //指定deviceID抽取最大值
+ {"type":"INF_MAX"} //指定deviceID抽取最大值
+ }，
+ // 用户指定的切分点，如果指定了切分点，Job将按照begin、end和split进行Task的切分，
+ // 切分的列只能是Partition Key（ParimaryKey的第一列）
+ // 支持INF_MIN, INF_MAX, STRING, INT
+ "split":[
+ {"type":"STRING", "value":"1"},
+ {"type":"STRING", "value":"2"},
+ {"type":"STRING", "value":"3"},
+ {"type":"STRING", "value":"4"},
+ {"type":"STRING", "value":"5"}
+ ]
+ }
+ ```
- * 必选：否
+ * 必选：否
- * 默认值：无
+ * 默认值：无
### 3.3 类型转换
@@ -291,44 +515,14 @@ OTSReader会根据OTS的表范围，按照Datax并发的数目N，将范围等
* 注意，OTS本身不支持日期型类型。应用层一般使用Long报错时间的Unix TimeStamp。
-## 4 性能报告
-### 4.1 环境准备
+## 4 约束限制
-#### 4.1.1 数据特征
-
-15列String(10 Byte), 2两列Integer(8 Byte)，总计168Byte/r。
-
-#### 4.1.2 机器参数
-
-OTS端：3台前端机，5台后端机
-
-DataX运行端: 24核CPU， 98GB内存
-
-#### 4.1.3 DataX jvm 参数
-
- -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError
-
-### 4.2 测试报告
-
-#### 4.2.1 测试报告
-
-|并发数|DataX CPU|OTS 流量|DATAX流量 | 前端QPS| 前端延时|
-|--------|--------| --------|--------|--------|------|
-|2| 36% |6.3M/s |12739 rec/s | 4.7 | 308ms |
-|11| 155% | 32M/s |60732 rec/s | 23.9 | 412ms |
-|50| 377% | 73M/s |145139 rec/s | 54 | 874ms |
-|100| 448% | 82M/s | 156262 rec/s |60 | 1570ms |
-
-
-
-## 5 约束限制
-
-### 5.1 一致性约束
+### 4.1 一致性约束
OTS是类BigTable的存储系统，OTS本身能够保证单行写事务性，无法提供跨行级别的事务。对于OTSReader而言也无法提供全表的一致性视图。例如对于OTSReader在0点启动的数据同步任务，在整个表数据同步过程中，OTSReader同样会抽取到后续更新的数据，无法提供准确的0点时刻该表一致性视图。
-### 5.2 增量数据同步
+### 4.2 增量数据同步
OTS本质上KV存储，目前只能针对PK进行范围查询，暂不支持按照字段范围抽取数据。因此只能对于增量查询，如果PK能够表示范围信息，例如自增ID，或者时间戳。
@@ -336,5 +530,4 @@ OTS本质上KV存储，目前只能针对PK进行范围查询，暂不支持按
时间戳， OTSReader可以通过PK过滤时间戳，通过制定Range范围进行增量抽取。这样使用的前提是OTS中的PrimaryKey必须包含主键时间列(时间主键需要使用OTS应用方生成。)
-## 6 FAQ
-
+## 5 FAQ
diff --git a/otsreader/pom.xml b/otsreader/pom.xml
index eaac8804..dad538bf 100644
--- a/otsreader/pom.xml
+++ b/otsreader/pom.xml
@@ -1,5 +1,5 @@
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4.0.0
com.alibaba.datax
@@ -10,17 +10,6 @@
otsreader
-
- org.apache.logging.log4j
- log4j-api
- 2.17.1
-
-
-
- org.apache.logging.log4j
- log4j-core
- 2.17.1
-
com.alibaba.datax
datax-common
@@ -47,22 +36,43 @@
2.2.4
- log4j-api
+ log4j-core
org.apache.logging.log4j
+
+
+
+ com.aliyun.openservices
+ tablestore
+ 5.13.13
+
log4j-core
org.apache.logging.log4j
-
+
com.google.code.gson
gson
2.2.4
+
+ com.alibaba
+ fastjson
+ 1.2.83_noneautotype
+ compile
+
+
+
+ src/main/java
+
+ **/*.properties
+
+
+
@@ -98,10 +108,6 @@
maven-surefire-plugin
2.5
- all
- 10
- true
- -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=.
**/unittest/*.java
**/functiontest/*.java
@@ -111,4 +117,3 @@
-
diff --git a/otsreader/src/main/assembly/package.xml b/otsreader/src/main/assembly/package.xml
index 7ee305d1..cb90f3e8 100644
--- a/otsreader/src/main/assembly/package.xml
+++ b/otsreader/src/main/assembly/package.xml
@@ -12,8 +12,8 @@
src/main/resources
plugin.json
- plugin_job_template.json
-
+ plugin_job_template.json
+
plugin/reader/otsreader
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/IOtsReaderMasterProxy.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/IOtsReaderMasterProxy.java
new file mode 100644
index 00000000..ee622e16
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/IOtsReaderMasterProxy.java
@@ -0,0 +1,15 @@
+package com.alibaba.datax.plugin.reader.otsreader;
+
+import java.util.List;
+
+import com.alibaba.datax.common.util.Configuration;
+
+public interface IOtsReaderMasterProxy {
+
+ public void init(Configuration param) throws Exception;
+
+ public List split(int num) throws Exception;
+
+ public void close();
+
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/IOtsReaderSlaveProxy.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/IOtsReaderSlaveProxy.java
new file mode 100644
index 00000000..d1100a2a
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/IOtsReaderSlaveProxy.java
@@ -0,0 +1,26 @@
+package com.alibaba.datax.plugin.reader.otsreader;
+
+import com.alibaba.datax.common.plugin.RecordSender;
+import com.alibaba.datax.common.util.Configuration;
+
+/**
+ * OTS Reader工作进程接口
+ */
+public interface IOtsReaderSlaveProxy {
+ /**
+ * 初始化函数，解析配置、初始化相关资源
+ */
+ public void init(Configuration configuration);
+
+ /**
+ * 关闭函数，释放资源
+ */
+ public void close();
+
+ /**
+ * 数据导出函数
+ * @param recordSender
+ * @throws Exception
+ */
+ public void startRead(RecordSender recordSender) throws Exception;
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReader.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReader.java
index 8880c07e..c6bc44b8 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReader.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReader.java
@@ -1,45 +1,48 @@
package com.alibaba.datax.plugin.reader.otsreader;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordSender;
import com.alibaba.datax.common.spi.Reader;
import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.datax.plugin.reader.otsreader.utils.Common;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSConf;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSMode;
+import com.alibaba.datax.plugin.reader.otsreader.utils.Constant;
+import com.alibaba.datax.plugin.reader.otsreader.utils.GsonParser;
+import com.alibaba.datax.plugin.reader.otsreader.utils.OtsReaderError;
+import com.alicloud.openservices.tablestore.TableStoreException;
import com.aliyun.openservices.ots.ClientException;
-import com.aliyun.openservices.ots.OTSException;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
public class OtsReader extends Reader {
public static class Job extends Reader.Job {
private static final Logger LOG = LoggerFactory.getLogger(Job.class);
- private OtsReaderMasterProxy proxy = new OtsReaderMasterProxy();
+ //private static final MessageSource MESSAGE_SOURCE = MessageSource.loadResourceBundle(OtsReader.class);
+ private IOtsReaderMasterProxy proxy = null;
+
@Override
- public void init() {
+ public void init() {
LOG.info("init() begin ...");
+
+ proxy = new OtsReaderMasterProxy();
try {
this.proxy.init(getPluginJobConf());
- } catch (OTSException e) {
- LOG.error("OTSException. ErrorCode:{}, ErrorMsg:{}, RequestId:{}",
- new Object[]{e.getErrorCode(), e.getMessage(), e.getRequestId()});
- LOG.error("Stack", e);
- throw DataXException.asDataXException(new OtsReaderError(e.getErrorCode(), "OTS端的错误"), Common.getDetailMessage(e), e);
+ } catch (TableStoreException e) {
+ LOG.error("OTSException: {}", e.toString(), e);
+ throw DataXException.asDataXException(new OtsReaderError(e.getErrorCode(), "OTS ERROR"), e.toString(), e);
} catch (ClientException e) {
- LOG.error("ClientException. ErrorCode:{}, ErrorMsg:{}",
- new Object[]{e.getErrorCode(), e.getMessage()});
- LOG.error("Stack", e);
- throw DataXException.asDataXException(new OtsReaderError(e.getErrorCode(), "OTS端的错误"), Common.getDetailMessage(e), e);
- } catch (IllegalArgumentException e) {
- LOG.error("IllegalArgumentException. ErrorMsg:{}", e.getMessage(), e);
- throw DataXException.asDataXException(OtsReaderError.INVALID_PARAM, Common.getDetailMessage(e), e);
+ LOG.error("ClientException: {}", e.toString(), e);
+ throw DataXException.asDataXException(OtsReaderError.ERROR, e.toString(), e);
} catch (Exception e) {
- LOG.error("Exception. ErrorMsg:{}", e.getMessage(), e);
- throw DataXException.asDataXException(OtsReaderError.ERROR, Common.getDetailMessage(e), e);
+ LOG.error("Exception. ErrorMsg:{}", e.toString(), e);
+ throw DataXException.asDataXException(OtsReaderError.ERROR, e.toString(), e);
}
+
LOG.info("init() end ...");
}
@@ -60,22 +63,9 @@ public class OtsReader extends Reader {
try {
confs = this.proxy.split(adviceNumber);
- } catch (OTSException e) {
- LOG.error("OTSException. ErrorCode:{}, ErrorMsg:{}, RequestId:{}",
- new Object[]{e.getErrorCode(), e.getMessage(), e.getRequestId()});
- LOG.error("Stack", e);
- throw DataXException.asDataXException(new OtsReaderError(e.getErrorCode(), "OTS端的错误"), Common.getDetailMessage(e), e);
- } catch (ClientException e) {
- LOG.error("ClientException. ErrorCode:{}, ErrorMsg:{}",
- new Object[]{e.getErrorCode(), e.getMessage()});
- LOG.error("Stack", e);
- throw DataXException.asDataXException(new OtsReaderError(e.getErrorCode(), "OTS端的错误"), Common.getDetailMessage(e), e);
- } catch (IllegalArgumentException e) {
- LOG.error("IllegalArgumentException. ErrorMsg:{}", e.getMessage(), e);
- throw DataXException.asDataXException(OtsReaderError.INVALID_PARAM, Common.getDetailMessage(e), e);
} catch (Exception e) {
LOG.error("Exception. ErrorMsg:{}", e.getMessage(), e);
- throw DataXException.asDataXException(OtsReaderError.ERROR, Common.getDetailMessage(e), e);
+ throw DataXException.asDataXException(OtsReaderError.ERROR, e.toString(), e);
}
LOG.info("split() end ...");
@@ -85,39 +75,60 @@ public class OtsReader extends Reader {
public static class Task extends Reader.Task {
private static final Logger LOG = LoggerFactory.getLogger(Task.class);
- private OtsReaderSlaveProxy proxy = new OtsReaderSlaveProxy();
+ //private static final MessageSource MESSAGE_SOURCE = MessageSource.loadResourceBundle(OtsReader.class);
+ private IOtsReaderSlaveProxy proxy = null;
@Override
public void init() {
+
+ OTSConf conf = GsonParser.jsonToConf((String) this.getPluginJobConf().get(Constant.ConfigKey.CONF));
+ // 是否使用新接口
+ if(conf.isNewVersion()) {
+ if (conf.getMode() == OTSMode.MULTI_VERSION) {
+ LOG.info("init OtsReaderSlaveProxyMultiVersion");
+ proxy = new OtsReaderSlaveMultiVersionProxy();
+ } else {
+ LOG.info("init OtsReaderSlaveProxyNormal");
+ proxy = new OtsReaderSlaveNormalProxy();
+ }
+
+ }
+ else{
+ String metaMode = conf.getMetaMode();
+ if (StringUtils.isNotBlank(metaMode) && !metaMode.equalsIgnoreCase("false")) {
+ LOG.info("init OtsMetaReaderSlaveProxy");
+ proxy = new OtsReaderSlaveMetaProxy();
+ } else {
+ LOG.info("init OtsReaderSlaveProxyOld");
+ proxy = new OtsReaderSlaveProxyOld();
+ }
+ }
+
+ proxy.init(this.getPluginJobConf());
}
@Override
public void destroy() {
+ try {
+ proxy.close();
+ } catch (Exception e) {
+ LOG.error("Exception. ErrorMsg:{}", e.toString(), e);
+ throw DataXException.asDataXException(OtsReaderError.ERROR, e.toString(), e);
+ }
}
@Override
public void startRead(RecordSender recordSender) {
- LOG.info("startRead() begin ...");
+
try {
- this.proxy.read(recordSender,getPluginJobConf());
- } catch (OTSException e) {
- LOG.error("OTSException. ErrorCode:{}, ErrorMsg:{}, RequestId:{}",
- new Object[]{e.getErrorCode(), e.getMessage(), e.getRequestId()});
- LOG.error("Stack", e);
- throw DataXException.asDataXException(new OtsReaderError(e.getErrorCode(), "OTS端的错误"), Common.getDetailMessage(e), e);
- } catch (ClientException e) {
- LOG.error("ClientException. ErrorCode:{}, ErrorMsg:{}",
- new Object[]{e.getErrorCode(), e.getMessage()});
- LOG.error("Stack", e);
- throw DataXException.asDataXException(new OtsReaderError(e.getErrorCode(), "OTS端的错误"), Common.getDetailMessage(e), e);
- } catch (IllegalArgumentException e) {
- LOG.error("IllegalArgumentException. ErrorMsg:{}", e.getMessage(), e);
- throw DataXException.asDataXException(OtsReaderError.INVALID_PARAM, Common.getDetailMessage(e), e);
+ proxy.startRead(recordSender);
} catch (Exception e) {
- LOG.error("Exception. ErrorMsg:{}", e.getMessage(), e);
- throw DataXException.asDataXException(OtsReaderError.ERROR, Common.getDetailMessage(e), e);
+ LOG.error("Exception. ErrorMsg:{}", e.toString(), e);
+ throw DataXException.asDataXException(OtsReaderError.ERROR, e.toString(), e);
}
- LOG.info("startRead() end ...");
+
+
+
}
}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderMasterProxy.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderMasterProxy.java
index 2b758f06..4ecdd8c1 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderMasterProxy.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderMasterProxy.java
@@ -1,221 +1,243 @@
package com.alibaba.datax.plugin.reader.otsreader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.reader.otsreader.callable.GetFirstRowPrimaryKeyCallable;
-import com.alibaba.datax.plugin.reader.otsreader.callable.GetTableMetaCallable;
import com.alibaba.datax.plugin.reader.otsreader.model.OTSConf;
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSConst;
import com.alibaba.datax.plugin.reader.otsreader.model.OTSRange;
-import com.alibaba.datax.plugin.reader.otsreader.utils.ParamChecker;
-import com.alibaba.datax.plugin.reader.otsreader.utils.Common;
-import com.alibaba.datax.plugin.reader.otsreader.utils.GsonParser;
-import com.alibaba.datax.plugin.reader.otsreader.utils.ReaderModelParser;
-import com.alibaba.datax.plugin.reader.otsreader.utils.RangeSplit;
-import com.alibaba.datax.plugin.reader.otsreader.utils.RetryHelper;
-import com.aliyun.openservices.ots.OTSClient;
-import com.aliyun.openservices.ots.model.Direction;
-import com.aliyun.openservices.ots.model.PrimaryKeyValue;
-import com.aliyun.openservices.ots.model.RangeRowQueryCriteria;
-import com.aliyun.openservices.ots.model.RowPrimaryKey;
-import com.aliyun.openservices.ots.model.TableMeta;
+import com.alibaba.datax.plugin.reader.otsreader.utils.*;
+import com.alicloud.openservices.tablestore.SyncClientInterface;
+import com.alicloud.openservices.tablestore.model.*;
+import com.alicloud.openservices.tablestore.model.timeseries.ScanTimeseriesDataResponse;
+import com.alicloud.openservices.tablestore.model.timeseries.TimeseriesScanSplitInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class OtsReaderMasterProxy {
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
- private OTSConf conf = new OTSConf();
-
- private OTSRange range = null;
-
- private OTSClient ots = null;
-
- private TableMeta meta = null;
-
- private Direction direction = null;
+public class OtsReaderMasterProxy implements IOtsReaderMasterProxy {
private static final Logger LOG = LoggerFactory.getLogger(OtsReaderMasterProxy.class);
+ private OTSConf conf = null;
+ private TableMeta meta = null;
+ private SyncClientInterface ots = null;
+ private Direction direction = null;
- /**
- * 1.检查参数是否为
- * null，endpoint,accessid,accesskey,instance-name,table,column,range-begin,range-end,range-split
- * 2.检查参数是否为空字符串
- * endpoint,accessid,accesskey,instance-name,table
- * 3.检查是否为空数组
- * column
- * 4.检查Range的类型个个数是否和PrimaryKey匹配
- * column,range-begin,range-end
- * 5.检查Range Split 顺序和类型是否Range一致，类型是否于PartitionKey一致
- * column-split
- * @param param
- * @throws Exception
- */
- public void init(Configuration param) throws Exception {
- // 默认参数
- // 每次重试的时间都是上一次的一倍，当sleep时间大于30秒时，Sleep重试时间不在增长。18次能覆盖OTS的Failover时间5分钟
- conf.setRetry(param.getInt(OTSConst.RETRY, 18));
- conf.setSleepInMilliSecond(param.getInt(OTSConst.SLEEP_IN_MILLI_SECOND, 100));
-
- // 必选参数
- conf.setEndpoint(ParamChecker.checkStringAndGet(param, Key.OTS_ENDPOINT));
- conf.setAccessId(ParamChecker.checkStringAndGet(param, Key.OTS_ACCESSID));
- conf.setAccesskey(ParamChecker.checkStringAndGet(param, Key.OTS_ACCESSKEY));
- conf.setInstanceName(ParamChecker.checkStringAndGet(param, Key.OTS_INSTANCE_NAME));
- conf.setTableName(ParamChecker.checkStringAndGet(param, Key.TABLE_NAME));
-
- ots = new OTSClient(
- this.conf.getEndpoint(),
- this.conf.getAccessId(),
- this.conf.getAccesskey(),
- this.conf.getInstanceName());
-
- meta = getTableMeta(ots, conf.getTableName());
- LOG.info("Table Meta : {}", GsonParser.metaToJson(meta));
-
- conf.setColumns(ReaderModelParser.parseOTSColumnList(ParamChecker.checkListAndGet(param, Key.COLUMN, true)));
-
- Map rangeMap = ParamChecker.checkMapAndGet(param, Key.RANGE, true);
- conf.setRangeBegin(ReaderModelParser.parsePrimaryKey(ParamChecker.checkListAndGet(rangeMap, Key.RANGE_BEGIN, false)));
- conf.setRangeEnd(ReaderModelParser.parsePrimaryKey(ParamChecker.checkListAndGet(rangeMap, Key.RANGE_END, false)));
-
- range = ParamChecker.checkRangeAndGet(meta, this.conf.getRangeBegin(), this.conf.getRangeEnd());
-
- direction = ParamChecker.checkDirectionAndEnd(meta, range.getBegin(), range.getEnd());
- LOG.info("Direction : {}", direction);
-
- List points = ReaderModelParser.parsePrimaryKey(ParamChecker.checkListAndGet(rangeMap, Key.RANGE_SPLIT));
- ParamChecker.checkInputSplitPoints(meta, range, direction, points);
- conf.setRangeSplit(points);
- }
-
- public List split(int num) throws Exception {
- LOG.info("Expect split num : " + num);
-
- List configurations = new ArrayList();
-
- List ranges = null;
-
- if (this.conf.getRangeSplit() != null) { // 用户显示指定了拆分范围
- LOG.info("Begin userDefinedRangeSplit");
- ranges = userDefinedRangeSplit(meta, range, this.conf.getRangeSplit());
- LOG.info("End userDefinedRangeSplit");
- } else { // 采用默认的切分算法
- LOG.info("Begin defaultRangeSplit");
- ranges = defaultRangeSplit(ots, meta, range, num);
- LOG.info("End defaultRangeSplit");
- }
-
- // 解决大量的Split Point序列化消耗内存的问题
- // 因为slave中不会使用这个配置，所以置为空
- this.conf.setRangeSplit(null);
-
- for (OTSRange item : ranges) {
- Configuration configuration = Configuration.newDefault();
- configuration.set(OTSConst.OTS_CONF, GsonParser.confToJson(this.conf));
- configuration.set(OTSConst.OTS_RANGE, GsonParser.rangeToJson(item));
- configuration.set(OTSConst.OTS_DIRECTION, GsonParser.directionToJson(direction));
- configurations.add(configuration);
- }
-
- LOG.info("Configuration list count : " + configurations.size());
-
- return configurations;
- }
public OTSConf getConf() {
return conf;
}
+ public TableMeta getMeta() {
+ return meta;
+ }
+
+ public SyncClientInterface getOts() {
+ return ots;
+ }
+
+ public void setOts(SyncClientInterface ots) {
+ this.ots = ots;
+ }
+
+ /**
+ * 基于配置传入的配置文件，解析为对应的参数
+ *
+ * @param param
+ * @throws Exception
+ */
+ public void init(Configuration param) throws Exception {
+ // 基于预定义的Json格式,检查传入参数是否符合Conf定义规范
+ conf = OTSConf.load(param);
+
+ // Init ots
+ ots = OtsHelper.getOTSInstance(conf);
+
+ // 宽行表init
+ if (!conf.isTimeseriesTable()) {
+ // 获取TableMeta
+ meta = OtsHelper.getTableMeta(
+ ots,
+ conf.getTableName(),
+ conf.getRetry(),
+ conf.getRetryPauseInMillisecond());
+
+ // 基于Meta检查Conf是否正确
+ ParamChecker.checkAndSetOTSConf(conf, meta);
+ direction = ParamChecker.checkDirectionAndEnd(meta, conf.getRange().getBegin(), conf.getRange().getEnd());
+ }
+ // 时序表 检查tablestore SDK version
+ if (conf.isTimeseriesTable()){
+ Common.checkTableStoreSDKVersion();
+ }
+
+
+ }
+
+ public List split(int mandatoryNumber) throws Exception {
+ LOG.info("Expect split num : " + mandatoryNumber);
+
+ List configurations = new ArrayList();
+
+ if (conf.isTimeseriesTable()) {{ // 时序表全部采用默认切分策略
+ LOG.info("Begin timeseries table defaultRangeSplit");
+ configurations = getTimeseriesConfigurationBySplit(mandatoryNumber);
+ LOG.info("End timeseries table defaultRangeSplit");
+ }}
+ else if (this.conf.getRange().getSplit().size() != 0) { // 用户显示指定了拆分范围
+ LOG.info("Begin userDefinedRangeSplit");
+ configurations = getNormalConfigurationBySplit();
+ LOG.info("End userDefinedRangeSplit");
+ } else { // 采用默认的切分算法
+ LOG.info("Begin defaultRangeSplit");
+ configurations = getDefaultConfiguration(mandatoryNumber);
+ LOG.info("End defaultRangeSplit");
+ }
+
+ LOG.info("Expect split num: "+ mandatoryNumber +", and final configuration list count : " + configurations.size());
+ return configurations;
+ }
+
public void close() {
ots.shutdown();
}
- // private function
-
- private TableMeta getTableMeta(OTSClient ots, String tableName) throws Exception {
- return RetryHelper.executeWithRetry(
- new GetTableMetaCallable(ots, tableName),
+ /**
+ * timeseries split信息，根据切分数配置多个Task
+ */
+ private List getTimeseriesConfigurationBySplit(int mandatoryNumber) throws Exception {
+ List timeseriesScanSplitInfoList = OtsHelper.splitTimeseriesScan(
+ ots,
+ conf.getTableName(),
+ conf.getMeasurementName(),
+ mandatoryNumber,
conf.getRetry(),
- conf.getSleepInMilliSecond()
- );
+ conf.getRetryPauseInMillisecond());
+ List configurations = new ArrayList<>();
+
+ for (int i = 0; i < timeseriesScanSplitInfoList.size(); i++) {
+ Configuration configuration = Configuration.newDefault();
+ configuration.set(Constant.ConfigKey.CONF, GsonParser.confToJson(conf));
+ configuration.set(Constant.ConfigKey.SPLIT_INFO, GsonParser.timeseriesScanSplitInfoToString(timeseriesScanSplitInfoList.get(i)));
+ configurations.add(configuration);
+ }
+ return configurations;
}
- private RowPrimaryKey getPKOfFirstRow(
- OTSRange range , Direction direction) throws Exception {
+ /**
+ * 根据用户配置的split信息，将配置文件基于Range范围转换为多个Task的配置
+ */
+ private List getNormalConfigurationBySplit() {
+ List> primaryKeys = new ArrayList>();
+ primaryKeys.add(conf.getRange().getBegin());
+ for (PrimaryKeyColumn column : conf.getRange().getSplit()) {
+ List point = new ArrayList();
+ point.add(column);
+ ParamChecker.fillPrimaryKey(this.meta.getPrimaryKeyList(), point, PrimaryKeyValue.INF_MIN);
+ primaryKeys.add(point);
+ }
+ primaryKeys.add(conf.getRange().getEnd());
- RangeRowQueryCriteria cur = new RangeRowQueryCriteria(this.conf.getTableName());
- cur.setInclusiveStartPrimaryKey(range.getBegin());
- cur.setExclusiveEndPrimaryKey(range.getEnd());
- cur.setLimit(1);
- cur.setColumnsToGet(Common.getPrimaryKeyNameList(meta));
- cur.setDirection(direction);
+ List configurations = new ArrayList(primaryKeys.size() - 1);
- return RetryHelper.executeWithRetry(
- new GetFirstRowPrimaryKeyCallable(ots, meta, cur),
- conf.getRetry(),
- conf.getSleepInMilliSecond()
- );
+ for (int i = 0; i < primaryKeys.size() - 1; i++) {
+ OTSRange range = new OTSRange();
+ range.setBegin(primaryKeys.get(i));
+ range.setEnd(primaryKeys.get(i + 1));
+
+ Configuration configuration = Configuration.newDefault();
+ configuration.set(Constant.ConfigKey.CONF, GsonParser.confToJson(conf));
+ configuration.set(Constant.ConfigKey.RANGE, GsonParser.rangeToJson(range));
+ configuration.set(Constant.ConfigKey.META, GsonParser.metaToJson(meta));
+ configurations.add(configuration);
+ }
+ return configurations;
}
- private List defaultRangeSplit(OTSClient ots, TableMeta meta, OTSRange range, int num) throws Exception {
+ private List getDefaultConfiguration(int num) throws Exception {
if (num == 1) {
List ranges = new ArrayList();
+ OTSRange range = new OTSRange();
+ range.setBegin(conf.getRange().getBegin());
+ range.setEnd(conf.getRange().getEnd());
ranges.add(range);
- return ranges;
+
+ return getConfigurationsFromRanges(ranges);
}
-
+
OTSRange reverseRange = new OTSRange();
- reverseRange.setBegin(range.getEnd());
- reverseRange.setEnd(range.getBegin());
+ reverseRange.setBegin(conf.getRange().getEnd());
+ reverseRange.setEnd(conf.getRange().getBegin());
Direction reverseDirection = (direction == Direction.FORWARD ? Direction.BACKWARD : Direction.FORWARD);
- RowPrimaryKey realBegin = getPKOfFirstRow(range, direction);
- RowPrimaryKey realEnd = getPKOfFirstRow(reverseRange, reverseDirection);
-
+ List realBegin = getPKOfFirstRow(conf.getRange(), direction);
+ List realEnd = getPKOfFirstRow(reverseRange, reverseDirection);
+
// 因为如果其中一行为空，表示这个范围内至多有一行数据
// 所以不再细分，直接使用用户定义的范围
if (realBegin == null || realEnd == null) {
List ranges = new ArrayList();
- ranges.add(range);
- return ranges;
+ ranges.add(conf.getRange());
+ return getConfigurationsFromRanges(ranges);
}
-
+
// 如果出现realBegin，realEnd的方向和direction不一致的情况，直接返回range
int cmp = Common.compareRangeBeginAndEnd(meta, realBegin, realEnd);
Direction realDirection = cmp > 0 ? Direction.BACKWARD : Direction.FORWARD;
if (realDirection != direction) {
LOG.warn("Expect '" + direction + "', but direction of realBegin and readlEnd is '" + realDirection + "'");
List ranges = new ArrayList();
- ranges.add(range);
- return ranges;
+ ranges.add(conf.getRange());
+ return getConfigurationsFromRanges(ranges);
}
List ranges = RangeSplit.rangeSplitByCount(meta, realBegin, realEnd, num);
if (ranges.isEmpty()) { // 当PartitionKey相等时，工具内部不会切分Range
- ranges.add(range);
+ ranges.add(conf.getRange());
} else {
// replace first and last
OTSRange first = ranges.get(0);
OTSRange last = ranges.get(ranges.size() - 1);
- first.setBegin(range.getBegin());
- last.setEnd(range.getEnd());
+ first.setBegin(conf.getRange().getBegin());
+ last.setEnd(conf.getRange().getEnd());
}
-
- return ranges;
+
+ return getConfigurationsFromRanges(ranges);
}
- private List userDefinedRangeSplit(TableMeta meta, OTSRange range, List points) {
- List ranges = RangeSplit.rangeSplitByPoint(meta, range.getBegin(), range.getEnd(), points);
- if (ranges.isEmpty()) { // 当PartitionKey相等时，工具内部不会切分Range
- ranges.add(range);
+ private List getConfigurationsFromRanges(List ranges){
+ List configurationList = new ArrayList<>();
+ for (OTSRange range:ranges
+ ) {
+ Configuration configuration = Configuration.newDefault();
+ configuration.set(Constant.ConfigKey.CONF, GsonParser.confToJson(conf));
+ configuration.set(Constant.ConfigKey.RANGE, GsonParser.rangeToJson(range));
+ configuration.set(Constant.ConfigKey.META, GsonParser.metaToJson(meta));
+ configurationList.add(configuration);
}
- return ranges;
+ return configurationList;
}
+
+ private List getPKOfFirstRow(
+ OTSRange range , Direction direction) throws Exception {
+
+ RangeRowQueryCriteria cur = new RangeRowQueryCriteria(this.conf.getTableName());
+ cur.setInclusiveStartPrimaryKey(new PrimaryKey(range.getBegin()));
+ cur.setExclusiveEndPrimaryKey(new PrimaryKey(range.getEnd()));
+ cur.setLimit(1);
+ cur.addColumnsToGet(Common.getPrimaryKeyNameList(meta));
+ cur.setDirection(direction);
+ cur.setMaxVersions(1);
+
+ return RetryHelper.executeWithRetry(
+ new GetFirstRowPrimaryKeyCallable(ots, meta, cur),
+ conf.getRetry(),
+ conf.getRetryPauseInMillisecond()
+ );
+ }
+
}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveMetaProxy.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveMetaProxy.java
new file mode 100644
index 00000000..f9860194
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveMetaProxy.java
@@ -0,0 +1,160 @@
+package com.alibaba.datax.plugin.reader.otsreader;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSConf;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSRange;
+import com.alibaba.datax.plugin.reader.otsreader.utils.Constant;
+import com.alibaba.datax.plugin.reader.otsreader.utils.Key;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.element.StringColumn;
+import com.alibaba.datax.common.plugin.RecordSender;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.reader.otsreader.utils.ParamCheckerOld;
+import com.alibaba.datax.plugin.reader.otsreader.utils.ReaderModelParser;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSColumn;
+import com.alibaba.datax.plugin.reader.otsreader.utils.DefaultNoRetry;
+import com.alibaba.datax.plugin.reader.otsreader.utils.GsonParser;
+import com.alibaba.fastjson.JSON;
+import com.aliyun.openservices.ots.OTSClient;
+import com.aliyun.openservices.ots.OTSServiceConfiguration;
+import com.aliyun.openservices.ots.model.DescribeTableRequest;
+import com.aliyun.openservices.ots.model.DescribeTableResult;
+import com.aliyun.openservices.ots.model.ListTableResult;
+import com.aliyun.openservices.ots.model.PrimaryKeyType;
+import com.aliyun.openservices.ots.model.ReservedThroughputDetails;
+import com.aliyun.openservices.ots.model.TableMeta;
+
+public class OtsReaderSlaveMetaProxy implements IOtsReaderSlaveProxy {
+
+ private OTSClient ots = null;
+ private OTSConf conf = null;
+ private OTSRange range = null;
+ private com.alicloud.openservices.tablestore.model.TableMeta meta = null;
+ private Configuration configuration = null;
+ private static final Logger LOG = LoggerFactory.getLogger(OtsReaderSlaveMetaProxy.class);
+
+
+ @Override
+ public void init(Configuration configuration) {
+ OTSServiceConfiguration configure = new OTSServiceConfiguration();
+ configure.setRetryStrategy(new DefaultNoRetry());
+
+ this.configuration = configuration;
+ conf = GsonParser.jsonToConf((String) configuration.get(Constant.ConfigKey.CONF));
+ range = GsonParser.jsonToRange((String) configuration.get(Constant.ConfigKey.RANGE));
+ meta = GsonParser.jsonToMeta((String) configuration.get(Constant.ConfigKey.META));
+
+ String endpoint = conf.getEndpoint();
+ String accessId = conf.getAccessId();
+ String accessKey = conf.getAccessKey();
+ String instanceName = conf.getInstanceName();
+
+ ots = new OTSClient(endpoint, accessId, accessKey, instanceName, null, configure, null);
+ }
+
+ @Override
+ public void close() {
+ ots.shutdown();
+ }
+
+ @Override
+ public void startRead(RecordSender recordSender) throws Exception {
+ List columns = ReaderModelParser
+ .parseOTSColumnList(ParamCheckerOld.checkListAndGet(configuration, Key.COLUMN, true));
+ String metaMode = conf.getMetaMode(); // column
+
+
+ ListTableResult listTableResult = null;
+ try {
+ listTableResult = ots.listTable();
+ LOG.info(String.format("ots listTable requestId:%s, traceId:%s", listTableResult.getRequestID(),
+ listTableResult.getTraceId()));
+ List allTables = listTableResult.getTableNames();
+ for (String eachTable : allTables) {
+ DescribeTableRequest describeTableRequest = new DescribeTableRequest();
+ describeTableRequest.setTableName(eachTable);
+ DescribeTableResult describeTableResult = ots.describeTable(describeTableRequest);
+ LOG.info(String.format("ots describeTable requestId:%s, traceId:%s", describeTableResult.getRequestID(),
+ describeTableResult.getTraceId()));
+
+ TableMeta tableMeta = describeTableResult.getTableMeta();
+ // table_name: first_table
+ // table primary key: type, data type: STRING
+ // table primary key: db_name, data type: STRING
+ // table primary key: table_name, data type: STRING
+ // Reserved throughput: read(0), write(0)
+ // last increase time: 1502881295
+ // last decrease time: None
+ // number of decreases today: 0
+
+ String tableName = tableMeta.getTableName();
+ Map primaryKey = tableMeta.getPrimaryKey();
+ ReservedThroughputDetails reservedThroughputDetails = describeTableResult
+ .getReservedThroughputDetails();
+ int reservedThroughputRead = reservedThroughputDetails.getCapacityUnit().getReadCapacityUnit();
+ int reservedThroughputWrite = reservedThroughputDetails.getCapacityUnit().getWriteCapacityUnit();
+ long lastIncreaseTime = reservedThroughputDetails.getLastIncreaseTime();
+ long lastDecreaseTime = reservedThroughputDetails.getLastDecreaseTime();
+ int numberOfDecreasesToday = reservedThroughputDetails.getNumberOfDecreasesToday();
+
+ Map allData = new HashMap();
+ allData.put("endpoint", conf.getEndpoint());
+ allData.put("instanceName", conf.getInstanceName());
+ allData.put("table", tableName);
+ // allData.put("primaryKey", JSON.toJSONString(primaryKey));
+ allData.put("reservedThroughputRead", reservedThroughputRead + "");
+ allData.put("reservedThroughputWrite", reservedThroughputWrite + "");
+ allData.put("lastIncreaseTime", lastIncreaseTime + "");
+ allData.put("lastDecreaseTime", lastDecreaseTime + "");
+ allData.put("numberOfDecreasesToday", numberOfDecreasesToday + "");
+
+ // 可扩展的可配置的形式
+ if ("column".equalsIgnoreCase(metaMode)) {
+ // 如果是列元数据模式并且column中配置的name是primaryKey，映射成多行DataX Record
+ List primaryKeyRecords = new ArrayList();
+ for (Entry eachPk : primaryKey.entrySet()) {
+ Record line = recordSender.createRecord();
+ for (OTSColumn col : columns) {
+ if (col.getColumnType() == OTSColumn.OTSColumnType.CONST) {
+ line.addColumn(col.getValue());
+ } else if ("primaryKey.name".equalsIgnoreCase(col.getName())) {
+ line.addColumn(new StringColumn(eachPk.getKey()));
+ } else if ("primaryKey.type".equalsIgnoreCase(col.getName())) {
+ line.addColumn(new StringColumn(eachPk.getValue().name()));
+ } else {
+ String v = allData.get(col.getName());
+ line.addColumn(new StringColumn(v));
+ }
+ }
+ LOG.debug("Reader send record : {}", line.toString());
+ recordSender.sendToWriter(line);
+ primaryKeyRecords.add(line);
+ }
+ } else {
+ Record line = recordSender.createRecord();
+ for (OTSColumn col : columns) {
+ if (col.getColumnType() == OTSColumn.OTSColumnType.CONST) {
+ line.addColumn(col.getValue());
+ } else {
+ String v = allData.get(col.getName());
+ line.addColumn(new StringColumn(v));
+ }
+ }
+ LOG.debug("Reader send record : {}", line.toString());
+ recordSender.sendToWriter(line);
+ }
+ }
+ } catch (Exception e) {
+ LOG.warn(JSON.toJSONString(listTableResult), e);
+ }
+
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveMultiVersionProxy.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveMultiVersionProxy.java
new file mode 100644
index 00000000..818a507e
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveMultiVersionProxy.java
@@ -0,0 +1,102 @@
+package com.alibaba.datax.plugin.reader.otsreader;
+
+import com.alibaba.datax.common.element.LongColumn;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.element.StringColumn;
+import com.alibaba.datax.common.plugin.RecordSender;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSConf;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSRange;
+import com.alibaba.datax.plugin.reader.otsreader.utils.*;
+import com.alicloud.openservices.tablestore.SyncClientInterface;
+import com.alicloud.openservices.tablestore.model.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class OtsReaderSlaveMultiVersionProxy implements IOtsReaderSlaveProxy {
+ private OTSConf conf = null;
+ private OTSRange range = null;
+ private TableMeta meta = null;
+ private SyncClientInterface ots = null;
+
+ private static final Logger LOG = LoggerFactory.getLogger(OtsReaderSlaveMultiVersionProxy.class);
+
+ @Override
+ public void init(Configuration configuration) {
+ conf = GsonParser.jsonToConf((String) configuration.get(Constant.ConfigKey.CONF));
+ range = GsonParser.jsonToRange((String) configuration.get(Constant.ConfigKey.RANGE));
+ meta = GsonParser.jsonToMeta((String) configuration.get(Constant.ConfigKey.META));
+
+ this.ots = OtsHelper.getOTSInstance(conf);
+ }
+
+ @Override
+ public void close() {
+ ots.shutdown();
+ }
+
+ private void sendToDatax(RecordSender recordSender, PrimaryKey pk, Column c) {
+ Record line = recordSender.createRecord();
+ //-------------------------
+ // 四元组 pk, column name, timestamp, value
+ //-------------------------
+
+ // pk
+ for( PrimaryKeyColumn pkc : pk.getPrimaryKeyColumns()) {
+ line.addColumn(TranformHelper.otsPrimaryKeyColumnToDataxColumn(pkc));
+ }
+ // column name
+ line.addColumn(new StringColumn(c.getName()));
+ // Timestamp
+ line.addColumn(new LongColumn(c.getTimestamp()));
+ // Value
+ line.addColumn(TranformHelper.otsColumnToDataxColumn(c));
+
+ recordSender.sendToWriter(line);
+ }
+
+ private void sendToDatax(RecordSender recordSender, Row row) {
+ PrimaryKey pk = row.getPrimaryKey();
+ for (Column c : row.getColumns()) {
+ sendToDatax(recordSender, pk, c);
+ }
+ }
+
+ /**
+ * 将获取到的数据采用4元组的方式传递给datax
+ * @param recordSender
+ * @param result
+ */
+ private void sendToDatax(RecordSender recordSender, GetRangeResponse result) {
+ LOG.debug("Per request get row count : " + result.getRows().size());
+ for (Row row : result.getRows()) {
+ sendToDatax(recordSender, row);
+ }
+ }
+
+ @Override
+ public void startRead(RecordSender recordSender) throws Exception {
+
+ PrimaryKey inclusiveStartPrimaryKey = new PrimaryKey(range.getBegin());
+ PrimaryKey exclusiveEndPrimaryKey = new PrimaryKey(range.getEnd());
+ PrimaryKey next = inclusiveStartPrimaryKey;
+
+ RangeRowQueryCriteria rangeRowQueryCriteria = new RangeRowQueryCriteria(conf.getTableName());
+ rangeRowQueryCriteria.setExclusiveEndPrimaryKey(exclusiveEndPrimaryKey);
+ rangeRowQueryCriteria.setDirection(Common.getDirection(range.getBegin(), range.getEnd()));
+ rangeRowQueryCriteria.setTimeRange(conf.getMulti().getTimeRange());
+ rangeRowQueryCriteria.setMaxVersions(conf.getMulti().getMaxVersion());
+ rangeRowQueryCriteria.addColumnsToGet(Common.toColumnToGet(conf.getColumn(), meta));
+
+ do{
+ rangeRowQueryCriteria.setInclusiveStartPrimaryKey(next);
+ GetRangeResponse result = OtsHelper.getRange(
+ ots,
+ rangeRowQueryCriteria,
+ conf.getRetry(),
+ conf.getRetryPauseInMillisecond());
+ sendToDatax(recordSender, result);
+ next = result.getNextStartPrimaryKey();
+ } while(next != null);
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveNormalProxy.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveNormalProxy.java
new file mode 100644
index 00000000..f7d89b15
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveNormalProxy.java
@@ -0,0 +1,256 @@
+package com.alibaba.datax.plugin.reader.otsreader;
+
+import com.alibaba.datax.common.element.LongColumn;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.element.StringColumn;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.common.plugin.RecordSender;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSColumn;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSConf;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSCriticalException;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSRange;
+import com.alibaba.datax.plugin.reader.otsreader.utils.*;
+import com.alicloud.openservices.tablestore.SyncClientInterface;
+import com.alicloud.openservices.tablestore.core.utils.Pair;
+import com.alicloud.openservices.tablestore.model.*;
+import com.alicloud.openservices.tablestore.model.timeseries.ScanTimeseriesDataRequest;
+import com.alicloud.openservices.tablestore.model.timeseries.ScanTimeseriesDataResponse;
+import com.alicloud.openservices.tablestore.model.timeseries.TimeseriesRow;
+import com.alicloud.openservices.tablestore.model.timeseries.TimeseriesScanSplitInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class OtsReaderSlaveNormalProxy implements IOtsReaderSlaveProxy {
+ private static final Logger LOG = LoggerFactory.getLogger(OtsReaderSlaveNormalProxy.class);
+ private OTSConf conf = null;
+ private OTSRange range = null;
+ private TableMeta meta = null;
+ private SyncClientInterface ots = null;
+ private TimeseriesScanSplitInfo splitInfo = null;
+
+ @Override
+ public void init(Configuration configuration) {
+ conf = GsonParser.jsonToConf((String) configuration.get(Constant.ConfigKey.CONF));
+ if (!conf.isTimeseriesTable()) {
+ range = GsonParser.jsonToRange((String) configuration.get(Constant.ConfigKey.RANGE));
+ meta = GsonParser.jsonToMeta((String) configuration.get(Constant.ConfigKey.META));
+ } else {
+ splitInfo = GsonParser.stringToTimeseriesScanSplitInfo((String) configuration.get(Constant.ConfigKey.SPLIT_INFO));
+ // 时序表 检查tablestore SDK version
+ try{
+ Common.checkTableStoreSDKVersion();
+ }
+ catch (Exception e){
+ LOG.error("Exception. ErrorMsg:{}", e.getMessage(), e);
+ throw DataXException.asDataXException(OtsReaderError.ERROR, e.toString(), e);
+ }
+ }
+
+
+ this.ots = OtsHelper.getOTSInstance(conf);
+ }
+
+ @Override
+ public void close() {
+ ots.shutdown();
+ }
+
+ private void sendToDatax(RecordSender recordSender, Row row) {
+ Record line = recordSender.createRecord();
+
+ PrimaryKey pk = row.getPrimaryKey();
+ for (OTSColumn column : conf.getColumn()) {
+ if (column.getColumnType() == OTSColumn.OTSColumnType.NORMAL) {
+ // 获取指定的列
+ PrimaryKeyColumn value = pk.getPrimaryKeyColumn(column.getName());
+ if (value != null) {
+ line.addColumn(TranformHelper.otsPrimaryKeyColumnToDataxColumn(value));
+ } else {
+ Column c = row.getLatestColumn(column.getName());
+ if (c != null) {
+ line.addColumn(TranformHelper.otsColumnToDataxColumn(c));
+ } else {
+ // 这里使用StringColumn的无参构造函数构造对象，而不是用null，下
+ // 游（writer）应该通过获取Column，然后通过Column的数据接口的返回值
+ // 是否是null来判断改Column是否为null
+ // Datax其他插件的也是使用这种方式，约定俗成，并没有使用直接向record中注入null方式代表空
+ line.addColumn(new StringColumn());
+ }
+ }
+ } else {
+ line.addColumn(column.getValue());
+ }
+ }
+ recordSender.sendToWriter(line);
+ }
+
+ private void sendToDatax(RecordSender recordSender, TimeseriesRow row) {
+
+
+ Record line = recordSender.createRecord();
+ // 对于配置项中的每一列
+ for (int i = 0; i < conf.getColumn().size(); i++) {
+ OTSColumn column = conf.getColumn().get(i);
+ // 如果不是常数列
+ if (column.getColumnType() == OTSColumn.OTSColumnType.NORMAL) {
+ // 如果是tags内字段
+ if (conf.getColumn().get(i).getTimeseriesTag()) {
+ String s = row.getTimeseriesKey().getTags().get(column.getName());
+ line.addColumn(new StringColumn(s));
+ }
+ // 如果为measurement字段
+ else if (column.getName().equals(Constant.ConfigKey.TimeseriesPKColumn.MEASUREMENT_NAME)) {
+ String s = row.getTimeseriesKey().getMeasurementName();
+ line.addColumn(new StringColumn(s));
+ }
+ // 如果为dataSource字段
+ else if (column.getName().equals(Constant.ConfigKey.TimeseriesPKColumn.DATA_SOURCE)) {
+ String s = row.getTimeseriesKey().getDataSource();
+ line.addColumn(new StringColumn(s));
+ }
+ // 如果为tags字段
+ else if (column.getName().equals(Constant.ConfigKey.TimeseriesPKColumn.TAGS)) {
+ line.addColumn(new StringColumn(row.getTimeseriesKey().buildTagsString()));
+ }
+ else if (column.getName().equals(Constant.ConfigKey.TimeseriesPKColumn.TIME)) {
+ Long l = row.getTimeInUs();
+ line.addColumn(new LongColumn(l));
+ }
+ // 否则为field内字段
+ else {
+ ColumnValue c = row.getFields().get(column.getName());
+ if (c == null) {
+ LOG.warn("Get column {} : type {} failed, use empty string instead", column.getName(), conf.getColumn().get(i).getValueType());
+ line.addColumn(new StringColumn());
+ } else if (c.getType() != conf.getColumn().get(i).getValueType()) {
+ LOG.warn("Get column {} failed, expected type: {}, actual type: {}. Sending actual type to writer.", column.getName(), conf.getColumn().get(i).getValueType(), c.getType());
+ line.addColumn(TranformHelper.otsColumnToDataxColumn(c));
+ } else {
+ line.addColumn(TranformHelper.otsColumnToDataxColumn(c));
+ }
+ }
+ }
+ // 如果是常数列
+ else {
+ line.addColumn(column.getValue());
+ }
+ }
+ recordSender.sendToWriter(line);
+ }
+
+ /**
+ * 将获取到的数据根据用户配置Column的方式传递给datax
+ *
+ * @param recordSender
+ * @param result
+ */
+ private void sendToDatax(RecordSender recordSender, GetRangeResponse result) {
+ for (Row row : result.getRows()) {
+ sendToDatax(recordSender, row);
+ }
+ }
+
+ private void sendToDatax(RecordSender recordSender, ScanTimeseriesDataResponse result) {
+ for (TimeseriesRow row : result.getRows()) {
+ sendToDatax(recordSender, row);
+ }
+ }
+
+ @Override
+ public void startRead(RecordSender recordSender) throws Exception {
+ if (conf.isTimeseriesTable()) {
+ readTimeseriesTable(recordSender);
+ } else {
+ readNormalTable(recordSender);
+ }
+ }
+
+ public void readTimeseriesTable(RecordSender recordSender) throws Exception {
+
+ List timeseriesPkName = new ArrayList<>();
+ timeseriesPkName.add(Constant.ConfigKey.TimeseriesPKColumn.MEASUREMENT_NAME);
+ timeseriesPkName.add(Constant.ConfigKey.TimeseriesPKColumn.DATA_SOURCE);
+ timeseriesPkName.add(Constant.ConfigKey.TimeseriesPKColumn.TAGS);
+ timeseriesPkName.add(Constant.ConfigKey.TimeseriesPKColumn.TIME);
+
+ ScanTimeseriesDataRequest scanTimeseriesDataRequest = new ScanTimeseriesDataRequest(conf.getTableName());
+ List> fieldsToGet = new ArrayList<>();
+ for (int i = 0; i < conf.getColumn().size(); i++) {
+ /**
+ * 如果所配置列
+ * 1. 不是常量列（即列名不为null）
+ * 2. 列名不在["measurementName","dataSource","tags"]中
+ * 3. 不是tags内的字段
+ * 则为需要获取的field字段。
+ */
+ String fieldName = conf.getColumn().get(i).getName();
+ if (fieldName != null && !timeseriesPkName.contains(fieldName) && !conf.getColumn().get(i).getTimeseriesTag()) {
+ Pair pair = new Pair<>(fieldName, conf.getColumn().get(i).getValueType());
+ fieldsToGet.add(pair);
+ }
+ }
+ scanTimeseriesDataRequest.setFieldsToGet(fieldsToGet);
+ scanTimeseriesDataRequest.setSplitInfo(splitInfo);
+
+ while (true) {
+ ScanTimeseriesDataResponse response = OtsHelper.scanTimeseriesData(
+ ots,
+ scanTimeseriesDataRequest,
+ conf.getRetry(),
+ conf.getRetryPauseInMillisecond());
+ sendToDatax(recordSender, response);
+ if (response.getNextToken() == null) {
+ break;
+ }
+ scanTimeseriesDataRequest.setNextToken(response.getNextToken());
+ }
+ }
+
+ public void readNormalTable(RecordSender recordSender) throws Exception {
+ PrimaryKey inclusiveStartPrimaryKey = new PrimaryKey(range.getBegin());
+ PrimaryKey exclusiveEndPrimaryKey = new PrimaryKey(range.getEnd());
+ PrimaryKey next = inclusiveStartPrimaryKey;
+
+ RangeRowQueryCriteria rangeRowQueryCriteria = new RangeRowQueryCriteria(conf.getTableName());
+ rangeRowQueryCriteria.setExclusiveEndPrimaryKey(exclusiveEndPrimaryKey);
+ rangeRowQueryCriteria.setDirection(Common.getDirection(range.getBegin(), range.getEnd()));
+ rangeRowQueryCriteria.setMaxVersions(1);
+ rangeRowQueryCriteria.addColumnsToGet(Common.toColumnToGet(conf.getColumn(), meta));
+
+ do {
+ rangeRowQueryCriteria.setInclusiveStartPrimaryKey(next);
+ GetRangeResponse result = OtsHelper.getRange(
+ ots,
+ rangeRowQueryCriteria,
+ conf.getRetry(),
+ conf.getRetryPauseInMillisecond());
+ sendToDatax(recordSender, result);
+ next = result.getNextStartPrimaryKey();
+ } while (next != null);
+ }
+
+
+ public void setConf(OTSConf conf) {
+ this.conf = conf;
+ }
+
+
+ public void setRange(OTSRange range) {
+ this.range = range;
+ }
+
+
+ public void setMeta(TableMeta meta) {
+ this.meta = meta;
+ }
+
+
+ public void setOts(SyncClientInterface ots) {
+ this.ots = ots;
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveProxy.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveProxy.java
deleted file mode 100644
index e64b4e7e..00000000
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveProxy.java
+++ /dev/null
@@ -1,135 +0,0 @@
-package com.alibaba.datax.plugin.reader.otsreader;
-
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.datax.common.element.Record;
-import com.alibaba.datax.common.plugin.RecordSender;
-import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.datax.plugin.reader.otsreader.callable.GetRangeCallable;
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSColumn;
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSConf;
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSConst;
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSRange;
-import com.alibaba.datax.plugin.reader.otsreader.utils.Common;
-import com.alibaba.datax.plugin.reader.otsreader.utils.GsonParser;
-import com.alibaba.datax.plugin.reader.otsreader.utils.DefaultNoRetry;
-import com.alibaba.datax.plugin.reader.otsreader.utils.RetryHelper;
-import com.aliyun.openservices.ots.OTSClientAsync;
-import com.aliyun.openservices.ots.OTSServiceConfiguration;
-import com.aliyun.openservices.ots.model.Direction;
-import com.aliyun.openservices.ots.model.GetRangeRequest;
-import com.aliyun.openservices.ots.model.GetRangeResult;
-import com.aliyun.openservices.ots.model.OTSFuture;
-import com.aliyun.openservices.ots.model.RangeRowQueryCriteria;
-import com.aliyun.openservices.ots.model.Row;
-import com.aliyun.openservices.ots.model.RowPrimaryKey;
-
-public class OtsReaderSlaveProxy {
-
- class RequestItem {
- private RangeRowQueryCriteria criteria;
- private OTSFuture future;
-
- RequestItem(RangeRowQueryCriteria criteria, OTSFuture future) {
- this.criteria = criteria;
- this.future = future;
- }
-
- public RangeRowQueryCriteria getCriteria() {
- return criteria;
- }
-
- public OTSFuture getFuture() {
- return future;
- }
- }
-
- private static final Logger LOG = LoggerFactory.getLogger(OtsReaderSlaveProxy.class);
-
- private void rowsToSender(List rows, RecordSender sender, List columns) {
- for (Row row : rows) {
- Record line = sender.createRecord();
- line = Common.parseRowToLine(row, columns, line);
-
- LOG.debug("Reader send record : {}", line.toString());
-
- sender.sendToWriter(line);
- }
- }
-
- private RangeRowQueryCriteria generateRangeRowQueryCriteria(String tableName, RowPrimaryKey begin, RowPrimaryKey end, Direction direction, List columns) {
- RangeRowQueryCriteria criteria = new RangeRowQueryCriteria(tableName);
- criteria.setInclusiveStartPrimaryKey(begin);
- criteria.setDirection(direction);
- criteria.setColumnsToGet(columns);
- criteria.setLimit(-1);
- criteria.setExclusiveEndPrimaryKey(end);
- return criteria;
- }
-
- private RequestItem generateRequestItem(
- OTSClientAsync ots,
- OTSConf conf,
- RowPrimaryKey begin,
- RowPrimaryKey end,
- Direction direction,
- List columns) throws Exception {
- RangeRowQueryCriteria criteria = generateRangeRowQueryCriteria(conf.getTableName(), begin, end, direction, columns);
-
- GetRangeRequest request = new GetRangeRequest();
- request.setRangeRowQueryCriteria(criteria);
- OTSFuture future = ots.getRange(request);
-
- return new RequestItem(criteria, future);
- }
-
- public void read(RecordSender sender, Configuration configuration) throws Exception {
- LOG.info("read begin.");
-
- OTSConf conf = GsonParser.jsonToConf(configuration.getString(OTSConst.OTS_CONF));
- OTSRange range = GsonParser.jsonToRange(configuration.getString(OTSConst.OTS_RANGE));
- Direction direction = GsonParser.jsonToDirection(configuration.getString(OTSConst.OTS_DIRECTION));
-
- OTSServiceConfiguration configure = new OTSServiceConfiguration();
- configure.setRetryStrategy(new DefaultNoRetry());
-
- OTSClientAsync ots = new OTSClientAsync(
- conf.getEndpoint(),
- conf.getAccessId(),
- conf.getAccesskey(),
- conf.getInstanceName(),
- null,
- configure,
- null);
-
- RowPrimaryKey token = range.getBegin();
- List columns = Common.getNormalColumnNameList(conf.getColumns());
-
- RequestItem request = null;
-
- do {
- LOG.debug("Next token : {}", GsonParser.rowPrimaryKeyToJson(token));
- if (request == null) {
- request = generateRequestItem(ots, conf, token, range.getEnd(), direction, columns);
- } else {
- RequestItem req = request;
-
- GetRangeResult result = RetryHelper.executeWithRetry(
- new GetRangeCallable(ots, req.getCriteria(), req.getFuture()),
- conf.getRetry(),
- conf.getSleepInMilliSecond()
- );
- if ((token = result.getNextStartPrimaryKey()) != null) {
- request = generateRequestItem(ots, conf, token, range.getEnd(), direction, columns);
- }
-
- rowsToSender(result.getRows(), sender, conf.getColumns());
- }
- } while (token != null);
- ots.shutdown();
- LOG.info("read end.");
- }
-}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveProxyOld.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveProxyOld.java
new file mode 100644
index 00000000..72eb885e
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderSlaveProxyOld.java
@@ -0,0 +1,181 @@
+package com.alibaba.datax.plugin.reader.otsreader;
+
+import java.util.List;
+
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSRange;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSColumn;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSConf;
+import com.alibaba.datax.plugin.reader.otsreader.utils.*;
+import com.alicloud.openservices.tablestore.model.PrimaryKeyColumn;
+import com.aliyun.openservices.ots.model.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.plugin.RecordSender;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.reader.otsreader.callable.GetRangeCallableOld;
+import com.aliyun.openservices.ots.OTSClientAsync;
+import com.aliyun.openservices.ots.OTSServiceConfiguration;
+
+public class OtsReaderSlaveProxyOld implements IOtsReaderSlaveProxy {
+
+
+ private OTSClientAsync ots = null;
+ private OTSConf conf = null;
+ private OTSRange range = null;
+
+ class RequestItem {
+ private RangeRowQueryCriteria criteria;
+ private OTSFuture future;
+
+ RequestItem(RangeRowQueryCriteria criteria, OTSFuture future) {
+ this.criteria = criteria;
+ this.future = future;
+ }
+
+ public RangeRowQueryCriteria getCriteria() {
+ return criteria;
+ }
+
+ public OTSFuture getFuture() {
+ return future;
+ }
+ }
+
+ private static final Logger LOG = LoggerFactory.getLogger(OtsReaderSlaveProxyOld.class);
+
+ private void rowsToSender(List rows, RecordSender sender, List columns) {
+ for (Row row : rows) {
+ Record line = sender.createRecord();
+ line = CommonOld.parseRowToLine(row, columns, line);
+
+ LOG.debug("Reader send record : {}", line.toString());
+
+ sender.sendToWriter(line);
+ }
+ }
+
+ private RangeRowQueryCriteria generateRangeRowQueryCriteria(String tableName, RowPrimaryKey begin, RowPrimaryKey end, Direction direction, List columns) {
+ RangeRowQueryCriteria criteria = new RangeRowQueryCriteria(tableName);
+ criteria.setInclusiveStartPrimaryKey(begin);
+ criteria.setDirection(direction);
+ criteria.setColumnsToGet(columns);
+ criteria.setLimit(-1);
+ criteria.setExclusiveEndPrimaryKey(end);
+ return criteria;
+ }
+
+ private RequestItem generateRequestItem(
+ OTSClientAsync ots,
+ OTSConf conf,
+ RowPrimaryKey begin,
+ RowPrimaryKey end,
+ Direction direction,
+ List columns) throws Exception {
+ RangeRowQueryCriteria criteria = generateRangeRowQueryCriteria(conf.getTableName(), begin, end, direction, columns);
+
+ GetRangeRequest request = new GetRangeRequest();
+ request.setRangeRowQueryCriteria(criteria);
+ OTSFuture future = ots.getRange(request);
+
+ return new RequestItem(criteria, future);
+ }
+
+ @Override
+ public void init(Configuration configuration) {
+ conf = GsonParser.jsonToConf(configuration.getString(Constant.ConfigKey.CONF));
+ range = GsonParser.jsonToRange(configuration.getString(Constant.ConfigKey.RANGE));
+
+ OTSServiceConfiguration configure = new OTSServiceConfiguration();
+ configure.setRetryStrategy(new DefaultNoRetry());
+
+ ots = new OTSClientAsync(
+ conf.getEndpoint(),
+ conf.getAccessId(),
+ conf.getAccessKey(),
+ conf.getInstanceName(),
+ null,
+ configure,
+ null);
+ }
+
+ @Override
+ public void close() {
+ ots.shutdown();
+ }
+
+ @Override
+ public void startRead(RecordSender recordSender) throws Exception {
+ RowPrimaryKey token = pKColumnList2RowPrimaryKey(range.getBegin());
+
+ List columns = CommonOld.getNormalColumnNameList(conf.getColumn());
+ Direction direction = null;
+ switch (Common.getDirection(range.getBegin(), range.getEnd())){
+ case FORWARD:
+ direction = Direction.FORWARD;
+ break;
+ case BACKWARD:
+ default:
+ direction = Direction.BACKWARD;
+ }
+ RequestItem request = null;
+
+ do {
+ LOG.debug("Next token : {}", GsonParser.rowPrimaryKeyToJson(token));
+ if (request == null) {
+ request = generateRequestItem(ots, conf, token, pKColumnList2RowPrimaryKey(range.getEnd()), direction, columns);
+ } else {
+ RequestItem req = request;
+
+ GetRangeResult result = RetryHelperOld.executeWithRetry(
+ new GetRangeCallableOld(ots, req.getCriteria(), req.getFuture()),
+ conf.getRetry(),
+ // TODO
+ 100
+ );
+ if ((token = result.getNextStartPrimaryKey()) != null) {
+ request = generateRequestItem(ots, conf, token, pKColumnList2RowPrimaryKey(range.getEnd()), direction, columns);
+ }
+
+ rowsToSender(result.getRows(), recordSender, conf.getColumn());
+ }
+ } while (token != null);
+ }
+
+ /**
+ * 将 {@link com.alicloud.openservices.tablestore.model.PrimaryKeyColumn}的列表转为{@link com.aliyun.openservices.ots.model.RowPrimaryKey}
+ * @param list
+ * @return
+ */
+ public RowPrimaryKey pKColumnList2RowPrimaryKey(List list){
+ RowPrimaryKey rowPrimaryKey = new RowPrimaryKey();
+ for(PrimaryKeyColumn pk : list){
+ PrimaryKeyValue v = null;
+ if(pk.getValue() == com.alicloud.openservices.tablestore.model.PrimaryKeyValue.INF_MAX){
+ v = PrimaryKeyValue.INF_MAX;
+ } else if (pk.getValue() == com.alicloud.openservices.tablestore.model.PrimaryKeyValue.INF_MIN) {
+ v = PrimaryKeyValue.INF_MIN;
+ }
+ // 非INF_MAX 或 INF_MIN
+ else{
+ switch (pk.getValue().getType()){
+ case STRING:
+ v = PrimaryKeyValue.fromString(pk.getValue().asString());
+ break;
+ case INTEGER:
+ v = PrimaryKeyValue.fromLong(pk.getValue().asLong());
+ break;
+ case BINARY:
+ v = PrimaryKeyValue.fromBinary(pk.getValue().asBinary());
+ break;
+ default:
+ throw new IllegalArgumentException("the pKColumnList to RowPrimaryKey conversion failed");
+ }
+ }
+
+ rowPrimaryKey.addPrimaryKeyColumn(pk.getName(),v);
+ }
+ return rowPrimaryKey;
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/adaptor/ColumnAdaptor.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/adaptor/ColumnAdaptor.java
new file mode 100644
index 00000000..b2e14b5c
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/adaptor/ColumnAdaptor.java
@@ -0,0 +1,63 @@
+package com.alibaba.datax.plugin.reader.otsreader.adaptor;
+
+import com.alibaba.datax.common.element.*;
+import com.google.gson.*;
+import org.apache.commons.codec.binary.Base64;
+
+import java.lang.reflect.Type;
+
+public class ColumnAdaptor implements JsonDeserializer, JsonSerializer{
+ private final static String TYPE = "type";
+ private final static String RAW = "rawData";
+
+ @Override
+ public JsonElement serialize(Column obj, Type t,
+ JsonSerializationContext c) {
+ JsonObject json = new JsonObject();
+
+ String rawData = null;
+ switch (obj.getType()){
+ case BOOL:
+ rawData = String.valueOf(obj.getRawData()); break;
+ case BYTES:
+ rawData = Base64.encodeBase64String((byte[]) obj.getRawData()); break;
+ case DOUBLE:
+ rawData = String.valueOf(obj.getRawData());break;
+ case LONG:
+ rawData = String.valueOf(obj.getRawData());break;
+ case STRING:
+ rawData = String.valueOf(obj.getRawData());break;
+ default:
+ throw new IllegalArgumentException("Unsupport parse the column type:" + obj.getType().toString());
+
+ }
+ json.add(TYPE, new JsonPrimitive(obj.getType().toString()));
+ json.add(RAW, new JsonPrimitive(rawData));
+ return json;
+ }
+
+ @Override
+ public Column deserialize(JsonElement ele, Type t,
+ JsonDeserializationContext c) throws JsonParseException {
+ JsonObject obj = ele.getAsJsonObject();
+
+ String strType = obj.getAsJsonPrimitive(TYPE).getAsString();
+ String strRaw = obj.getAsJsonPrimitive(RAW).getAsString();
+ Column.Type type = Column.Type.valueOf(strType);
+ switch (type){
+ case BOOL:
+ return new BoolColumn(strRaw);
+ case BYTES:
+ return new BytesColumn(Base64.decodeBase64(strRaw));
+ case DOUBLE:
+ return new DoubleColumn(strRaw);
+ case LONG:
+ return new LongColumn(strRaw);
+ case STRING:
+ return new StringColumn(strRaw);
+ default:
+ throw new IllegalArgumentException("Unsupport parse the column type:" + type.toString());
+
+ }
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/adaptor/OTSColumnAdaptor.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/adaptor/OTSColumnAdaptor.java
deleted file mode 100644
index 25f9b682..00000000
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/adaptor/OTSColumnAdaptor.java
+++ /dev/null
@@ -1,117 +0,0 @@
-package com.alibaba.datax.plugin.reader.otsreader.adaptor;
-
-import java.lang.reflect.Type;
-
-import org.apache.commons.codec.binary.Base64;
-
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSColumn;
-import com.aliyun.openservices.ots.model.ColumnType;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-
-public class OTSColumnAdaptor implements JsonDeserializer, JsonSerializer{
- private final static String NAME = "name";
- private final static String COLUMN_TYPE = "column_type";
- private final static String VALUE_TYPE = "value_type";
- private final static String VALUE = "value";
-
- private void serializeConstColumn(JsonObject json, OTSColumn obj) {
- switch (obj.getValueType()) {
- case STRING :
- json.add(VALUE_TYPE, new JsonPrimitive(ColumnType.STRING.toString()));
- json.add(VALUE, new JsonPrimitive(obj.getValue().asString()));
- break;
- case INTEGER :
- json.add(VALUE_TYPE, new JsonPrimitive(ColumnType.INTEGER.toString()));
- json.add(VALUE, new JsonPrimitive(obj.getValue().asLong()));
- break;
- case DOUBLE :
- json.add(VALUE_TYPE, new JsonPrimitive(ColumnType.DOUBLE.toString()));
- json.add(VALUE, new JsonPrimitive(obj.getValue().asDouble()));
- break;
- case BOOLEAN :
- json.add(VALUE_TYPE, new JsonPrimitive(ColumnType.BOOLEAN.toString()));
- json.add(VALUE, new JsonPrimitive(obj.getValue().asBoolean()));
- break;
- case BINARY :
- json.add(VALUE_TYPE, new JsonPrimitive(ColumnType.BINARY.toString()));
- json.add(VALUE, new JsonPrimitive(Base64.encodeBase64String(obj.getValue().asBytes())));
- break;
- default:
- throw new IllegalArgumentException("Unsupport serialize the type : " + obj.getValueType() + "");
- }
- }
-
- private OTSColumn deserializeConstColumn(JsonObject obj) {
- String strType = obj.getAsJsonPrimitive(VALUE_TYPE).getAsString();
- ColumnType type = ColumnType.valueOf(strType);
-
- JsonPrimitive jsonValue = obj.getAsJsonPrimitive(VALUE);
-
- switch (type) {
- case STRING :
- return OTSColumn.fromConstStringColumn(jsonValue.getAsString());
- case INTEGER :
- return OTSColumn.fromConstIntegerColumn(jsonValue.getAsLong());
- case DOUBLE :
- return OTSColumn.fromConstDoubleColumn(jsonValue.getAsDouble());
- case BOOLEAN :
- return OTSColumn.fromConstBoolColumn(jsonValue.getAsBoolean());
- case BINARY :
- return OTSColumn.fromConstBytesColumn(Base64.decodeBase64(jsonValue.getAsString()));
- default:
- throw new IllegalArgumentException("Unsupport deserialize the type : " + type + "");
- }
- }
-
- private void serializeNormalColumn(JsonObject json, OTSColumn obj) {
- json.add(NAME, new JsonPrimitive(obj.getName()));
- }
-
- private OTSColumn deserializeNormarlColumn(JsonObject obj) {
- return OTSColumn.fromNormalColumn(obj.getAsJsonPrimitive(NAME).getAsString());
- }
-
- @Override
- public JsonElement serialize(OTSColumn obj, Type t,
- JsonSerializationContext c) {
- JsonObject json = new JsonObject();
-
- switch (obj.getColumnType()) {
- case CONST:
- json.add(COLUMN_TYPE, new JsonPrimitive(OTSColumn.OTSColumnType.CONST.toString()));
- serializeConstColumn(json, obj);
- break;
- case NORMAL:
- json.add(COLUMN_TYPE, new JsonPrimitive(OTSColumn.OTSColumnType.NORMAL.toString()));
- serializeNormalColumn(json, obj);
- break;
- default:
- throw new IllegalArgumentException("Unsupport serialize the type : " + obj.getColumnType() + "");
- }
- return json;
- }
-
- @Override
- public OTSColumn deserialize(JsonElement ele, Type t,
- JsonDeserializationContext c) throws JsonParseException {
- JsonObject obj = ele.getAsJsonObject();
- String strColumnType = obj.getAsJsonPrimitive(COLUMN_TYPE).getAsString();
- OTSColumn.OTSColumnType columnType = OTSColumn.OTSColumnType.valueOf(strColumnType);
-
- switch(columnType) {
- case CONST:
- return deserializeConstColumn(obj);
- case NORMAL:
- return deserializeNormarlColumn(obj);
- default:
- throw new IllegalArgumentException("Unsupport deserialize the type : " + columnType + "");
- }
- }
-}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/adaptor/PrimaryKeyValueAdaptor.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/adaptor/PrimaryKeyValueAdaptor.java
index 1a49ea47..240427ae 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/adaptor/PrimaryKeyValueAdaptor.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/adaptor/PrimaryKeyValueAdaptor.java
@@ -1,18 +1,12 @@
package com.alibaba.datax.plugin.reader.otsreader.adaptor;
-import java.lang.reflect.Type;
+import com.alicloud.openservices.tablestore.model.ColumnType;
+import com.alicloud.openservices.tablestore.model.PrimaryKeyType;
+import com.alicloud.openservices.tablestore.model.PrimaryKeyValue;
+import com.google.gson.*;
+import org.apache.commons.codec.binary.Base64;
-import com.aliyun.openservices.ots.model.ColumnType;
-import com.aliyun.openservices.ots.model.PrimaryKeyType;
-import com.aliyun.openservices.ots.model.PrimaryKeyValue;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
+import java.lang.reflect.Type;
/**
* {"type":"INF_MIN", "value":""}
@@ -31,27 +25,29 @@ public class PrimaryKeyValueAdaptor implements JsonDeserializer
JsonSerializationContext c) {
JsonObject json = new JsonObject();
- if (obj == PrimaryKeyValue.INF_MIN) {
+ if (obj.isInfMin()) {
json.add(TYPE, new JsonPrimitive(INF_MIN));
- json.add(VALUE, new JsonPrimitive(""));
return json;
}
- if (obj == PrimaryKeyValue.INF_MAX) {
+ if (obj.isInfMax()) {
json.add(TYPE, new JsonPrimitive(INF_MAX));
- json.add(VALUE, new JsonPrimitive(""));
return json;
}
switch (obj.getType()) {
case STRING :
- json.add(TYPE, new JsonPrimitive(ColumnType.STRING.toString()));
+ json.add(TYPE, new JsonPrimitive(ColumnType.STRING.toString()));
json.add(VALUE, new JsonPrimitive(obj.asString()));
break;
case INTEGER :
json.add(TYPE, new JsonPrimitive(ColumnType.INTEGER.toString()));
json.add(VALUE, new JsonPrimitive(obj.asLong()));
break;
+ case BINARY :
+ json.add(TYPE, new JsonPrimitive(ColumnType.BINARY.toString()));
+ json.add(VALUE, new JsonPrimitive(Base64.encodeBase64String(obj.asBinary())));
+ break;
default:
throw new IllegalArgumentException("Unsupport serialize the type : " + obj.getType() + "");
}
@@ -64,16 +60,17 @@ public class PrimaryKeyValueAdaptor implements JsonDeserializer
JsonObject obj = ele.getAsJsonObject();
String strType = obj.getAsJsonPrimitive(TYPE).getAsString();
- JsonPrimitive jsonValue = obj.getAsJsonPrimitive(VALUE);
- if (strType.equals(INF_MIN)) {
+ if (strType.equalsIgnoreCase(INF_MIN)) {
return PrimaryKeyValue.INF_MIN;
}
- if (strType.equals(INF_MAX)) {
+ if (strType.equalsIgnoreCase(INF_MAX)) {
return PrimaryKeyValue.INF_MAX;
}
+ JsonPrimitive jsonValue = obj.getAsJsonPrimitive(VALUE);
+
PrimaryKeyValue value = null;
PrimaryKeyType type = PrimaryKeyType.valueOf(strType);
switch(type) {
@@ -83,6 +80,9 @@ public class PrimaryKeyValueAdaptor implements JsonDeserializer
case INTEGER :
value = PrimaryKeyValue.fromLong(jsonValue.getAsLong());
break;
+ case BINARY :
+ value = PrimaryKeyValue.fromBinary(Base64.decodeBase64(jsonValue.getAsString()));
+ break;
default:
throw new IllegalArgumentException("Unsupport deserialize the type : " + type + "");
}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetFirstRowPrimaryKeyCallable.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetFirstRowPrimaryKeyCallable.java
index f004c0ff..cdcae91a 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetFirstRowPrimaryKeyCallable.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetFirstRowPrimaryKeyCallable.java
@@ -1,53 +1,42 @@
package com.alibaba.datax.plugin.reader.otsreader.callable;
+import com.alicloud.openservices.tablestore.SyncClientInterface;
+import com.alicloud.openservices.tablestore.model.*;
+
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
-import com.aliyun.openservices.ots.OTSClient;
-import com.aliyun.openservices.ots.model.ColumnType;
-import com.aliyun.openservices.ots.model.ColumnValue;
-import com.aliyun.openservices.ots.model.GetRangeRequest;
-import com.aliyun.openservices.ots.model.GetRangeResult;
-import com.aliyun.openservices.ots.model.PrimaryKeyType;
-import com.aliyun.openservices.ots.model.PrimaryKeyValue;
-import com.aliyun.openservices.ots.model.RangeRowQueryCriteria;
-import com.aliyun.openservices.ots.model.Row;
-import com.aliyun.openservices.ots.model.RowPrimaryKey;
-import com.aliyun.openservices.ots.model.TableMeta;
+public class GetFirstRowPrimaryKeyCallable implements Callable> {
-public class GetFirstRowPrimaryKeyCallable implements Callable{
-
- private OTSClient ots = null;
+ private SyncClientInterface ots = null;
private TableMeta meta = null;
private RangeRowQueryCriteria criteria = null;
-
- public GetFirstRowPrimaryKeyCallable(OTSClient ots, TableMeta meta, RangeRowQueryCriteria criteria) {
+
+ public GetFirstRowPrimaryKeyCallable(SyncClientInterface ots, TableMeta meta, RangeRowQueryCriteria criteria) {
this.ots = ots;
this.meta = meta;
this.criteria = criteria;
}
-
+
@Override
- public RowPrimaryKey call() throws Exception {
- RowPrimaryKey ret = new RowPrimaryKey();
+ public List call() throws Exception {
+ List ret = new ArrayList<>();
GetRangeRequest request = new GetRangeRequest();
request.setRangeRowQueryCriteria(criteria);
- GetRangeResult result = ots.getRange(request);
- List rows = result.getRows();
- if(rows.isEmpty()) {
+ GetRangeResponse response = ots.getRange(request);
+ List rows = response.getRows();
+ if (rows.isEmpty()) {
return null;// no data
- }
+ }
Row row = rows.get(0);
- Map pk = meta.getPrimaryKey();
- for (String key:pk.keySet()) {
- ColumnValue v = row.getColumns().get(key);
- if (v.getType() == ColumnType.INTEGER) {
- ret.addPrimaryKeyColumn(key, PrimaryKeyValue.fromLong(v.asLong()));
- } else {
- ret.addPrimaryKeyColumn(key, PrimaryKeyValue.fromString(v.asString()));
- }
+ Map pk = meta.getPrimaryKeyMap();
+
+ for (String key : pk.keySet()) {
+ PrimaryKeyColumn v = row.getPrimaryKey().getPrimaryKeyColumnsMap().get(key);
+ ret.add(v);
}
return ret;
}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetRangeCallable.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetRangeCallable.java
index 2cd1398a..995d491c 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetRangeCallable.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetRangeCallable.java
@@ -1,35 +1,26 @@
package com.alibaba.datax.plugin.reader.otsreader.callable;
+import com.alicloud.openservices.tablestore.SyncClientInterface;
+import com.alicloud.openservices.tablestore.model.GetRangeRequest;
+import com.alicloud.openservices.tablestore.model.GetRangeResponse;
+import com.alicloud.openservices.tablestore.model.RangeRowQueryCriteria;
+
import java.util.concurrent.Callable;
-import com.aliyun.openservices.ots.OTSClientAsync;
-import com.aliyun.openservices.ots.model.GetRangeRequest;
-import com.aliyun.openservices.ots.model.GetRangeResult;
-import com.aliyun.openservices.ots.model.OTSFuture;
-import com.aliyun.openservices.ots.model.RangeRowQueryCriteria;
-
-public class GetRangeCallable implements Callable {
+public class GetRangeCallable implements Callable {
- private OTSClientAsync ots;
+ private SyncClientInterface ots;
private RangeRowQueryCriteria criteria;
- private OTSFuture future;
- public GetRangeCallable(OTSClientAsync ots, RangeRowQueryCriteria criteria, OTSFuture future) {
+ public GetRangeCallable(SyncClientInterface ots, RangeRowQueryCriteria criteria) {
this.ots = ots;
this.criteria = criteria;
- this.future = future;
}
@Override
- public GetRangeResult call() throws Exception {
- try {
- return future.get();
- } catch (Exception e) {
- GetRangeRequest request = new GetRangeRequest();
- request.setRangeRowQueryCriteria(criteria);
- future = ots.getRange(request);
- throw e;
- }
+ public GetRangeResponse call() throws Exception {
+ GetRangeRequest request = new GetRangeRequest();
+ request.setRangeRowQueryCriteria(criteria);
+ return ots.getRange(request);
}
-
-}
+}
\ No newline at end of file
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetRangeCallableOld.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetRangeCallableOld.java
new file mode 100644
index 00000000..c0434126
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetRangeCallableOld.java
@@ -0,0 +1,35 @@
+package com.alibaba.datax.plugin.reader.otsreader.callable;
+
+import java.util.concurrent.Callable;
+
+import com.aliyun.openservices.ots.OTSClientAsync;
+import com.aliyun.openservices.ots.model.GetRangeRequest;
+import com.aliyun.openservices.ots.model.GetRangeResult;
+import com.aliyun.openservices.ots.model.OTSFuture;
+import com.aliyun.openservices.ots.model.RangeRowQueryCriteria;
+
+public class GetRangeCallableOld implements Callable {
+
+ private OTSClientAsync ots;
+ private RangeRowQueryCriteria criteria;
+ private OTSFuture future;
+
+ public GetRangeCallableOld(OTSClientAsync ots, RangeRowQueryCriteria criteria, OTSFuture future) {
+ this.ots = ots;
+ this.criteria = criteria;
+ this.future = future;
+ }
+
+ @Override
+ public GetRangeResult call() throws Exception {
+ try {
+ return future.get();
+ } catch (Exception e) {
+ GetRangeRequest request = new GetRangeRequest();
+ request.setRangeRowQueryCriteria(criteria);
+ future = ots.getRange(request);
+ throw e;
+ }
+ }
+
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetTableMetaCallable.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetTableMetaCallable.java
index 2884e12b..36a122c2 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetTableMetaCallable.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetTableMetaCallable.java
@@ -1,18 +1,19 @@
package com.alibaba.datax.plugin.reader.otsreader.callable;
+import com.alicloud.openservices.tablestore.SyncClientInterface;
+import com.alicloud.openservices.tablestore.model.DescribeTableRequest;
+import com.alicloud.openservices.tablestore.model.DescribeTableResponse;
+import com.alicloud.openservices.tablestore.model.TableMeta;
+
import java.util.concurrent.Callable;
-import com.aliyun.openservices.ots.OTSClient;
-import com.aliyun.openservices.ots.model.DescribeTableRequest;
-import com.aliyun.openservices.ots.model.DescribeTableResult;
-import com.aliyun.openservices.ots.model.TableMeta;
public class GetTableMetaCallable implements Callable{
- private OTSClient ots = null;
+ private SyncClientInterface ots = null;
private String tableName = null;
- public GetTableMetaCallable(OTSClient ots, String tableName) {
+ public GetTableMetaCallable(SyncClientInterface ots, String tableName) {
this.ots = ots;
this.tableName = tableName;
}
@@ -21,9 +22,9 @@ public class GetTableMetaCallable implements Callable{
public TableMeta call() throws Exception {
DescribeTableRequest describeTableRequest = new DescribeTableRequest();
describeTableRequest.setTableName(tableName);
- DescribeTableResult result = ots.describeTable(describeTableRequest);
+ DescribeTableResponse result = ots.describeTable(describeTableRequest);
TableMeta tableMeta = result.getTableMeta();
return tableMeta;
}
-}
+}
\ No newline at end of file
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetTimeseriesSplitCallable.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetTimeseriesSplitCallable.java
new file mode 100644
index 00000000..96521c41
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetTimeseriesSplitCallable.java
@@ -0,0 +1,38 @@
+package com.alibaba.datax.plugin.reader.otsreader.callable;
+
+import com.alicloud.openservices.tablestore.SyncClient;
+import com.alicloud.openservices.tablestore.SyncClientInterface;
+import com.alicloud.openservices.tablestore.TimeseriesClient;
+import com.alicloud.openservices.tablestore.model.timeseries.SplitTimeseriesScanTaskRequest;
+import com.alicloud.openservices.tablestore.model.timeseries.SplitTimeseriesScanTaskResponse;
+import com.alicloud.openservices.tablestore.model.timeseries.TimeseriesScanSplitInfo;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+
+public class GetTimeseriesSplitCallable implements Callable> {
+
+ private TimeseriesClient client = null;
+ private String timeseriesTableName = null;
+ private String measurementName = null;
+ private int splitCountHint = 1;
+
+
+ public GetTimeseriesSplitCallable(SyncClientInterface ots, String timeseriesTableName, String measurementName, int splitCountHint) {
+ this.client = ((SyncClient) ots).asTimeseriesClient();
+ this.timeseriesTableName = timeseriesTableName;
+ this.measurementName = measurementName;
+ this.splitCountHint = splitCountHint;
+ }
+
+ @Override
+ public List call() throws Exception {
+ SplitTimeseriesScanTaskRequest request = new SplitTimeseriesScanTaskRequest(timeseriesTableName, splitCountHint);
+ if (measurementName.length() != 0) {
+ request.setMeasurementName(measurementName);
+ }
+
+ SplitTimeseriesScanTaskResponse response = client.splitTimeseriesScanTask(request);
+ return response.getSplitInfos();
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/ScanTimeseriesDataCallable.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/ScanTimeseriesDataCallable.java
new file mode 100644
index 00000000..726d0e5d
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/ScanTimeseriesDataCallable.java
@@ -0,0 +1,27 @@
+package com.alibaba.datax.plugin.reader.otsreader.callable;
+
+import com.alicloud.openservices.tablestore.SyncClient;
+import com.alicloud.openservices.tablestore.SyncClientInterface;
+import com.alicloud.openservices.tablestore.TimeseriesClient;
+import com.alicloud.openservices.tablestore.model.timeseries.ScanTimeseriesDataRequest;
+import com.alicloud.openservices.tablestore.model.timeseries.ScanTimeseriesDataResponse;
+import com.alicloud.openservices.tablestore.model.timeseries.TimeseriesScanSplitInfo;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+
+public class ScanTimeseriesDataCallable implements Callable {
+
+ private TimeseriesClient client = null;
+ private ScanTimeseriesDataRequest request = null;
+
+ public ScanTimeseriesDataCallable(SyncClientInterface ots, ScanTimeseriesDataRequest scanTimeseriesDataRequest){
+ this.client = ((SyncClient) ots).asTimeseriesClient();
+ this.request = scanTimeseriesDataRequest;
+ }
+
+ @Override
+ public ScanTimeseriesDataResponse call() throws Exception {
+ return client.scanTimeseriesData(request);
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/DefaultNoRetry.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/DefaultNoRetry.java
new file mode 100644
index 00000000..b286472d
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/DefaultNoRetry.java
@@ -0,0 +1,32 @@
+package com.alibaba.datax.plugin.reader.otsreader.model;
+
+
+import com.alicloud.openservices.tablestore.model.DefaultRetryStrategy;
+import com.alicloud.openservices.tablestore.model.RetryStrategy;
+
+public class DefaultNoRetry extends DefaultRetryStrategy {
+
+ public DefaultNoRetry() {
+ super();
+ }
+
+ @Override
+ public RetryStrategy clone() {
+ return super.clone();
+ }
+
+ @Override
+ public int getRetries() {
+ return super.getRetries();
+ }
+
+ @Override
+ public boolean shouldRetry(String action, Exception ex) {
+ return false;
+ }
+
+ @Override
+ public long nextPause(String action, Exception ex) {
+ return super.nextPause(action, ex);
+ }
+}
\ No newline at end of file
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSColumn.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSColumn.java
index 129ccd2f..809f4c38 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSColumn.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSColumn.java
@@ -1,19 +1,18 @@
package com.alibaba.datax.plugin.reader.otsreader.model;
-import com.alibaba.datax.common.element.BoolColumn;
-import com.alibaba.datax.common.element.BytesColumn;
-import com.alibaba.datax.common.element.Column;
-import com.alibaba.datax.common.element.DoubleColumn;
-import com.alibaba.datax.common.element.LongColumn;
-import com.alibaba.datax.common.element.StringColumn;
-import com.aliyun.openservices.ots.model.ColumnType;
+import com.alibaba.datax.common.element.*;
+import com.alicloud.openservices.tablestore.model.ColumnType;
public class OTSColumn {
private String name;
private Column value;
+
private OTSColumnType columnType;
+
+ // 时序数据column配置
private ColumnType valueType;
-
+ private Boolean isTimeseriesTag;
+
public static enum OTSColumnType {
NORMAL, // 普通列
CONST // 常量列
@@ -24,10 +23,9 @@ public class OTSColumn {
this.columnType = OTSColumnType.NORMAL;
}
- private OTSColumn(Column value, ColumnType type) {
+ private OTSColumn(Column value) {
this.value = value;
this.columnType = OTSColumnType.CONST;
- this.valueType = type;
}
public static OTSColumn fromNormalColumn(String name) {
@@ -39,23 +37,23 @@ public class OTSColumn {
}
public static OTSColumn fromConstStringColumn(String value) {
- return new OTSColumn(new StringColumn(value), ColumnType.STRING);
+ return new OTSColumn(new StringColumn(value));
}
public static OTSColumn fromConstIntegerColumn(long value) {
- return new OTSColumn(new LongColumn(value), ColumnType.INTEGER);
+ return new OTSColumn(new LongColumn(value));
}
public static OTSColumn fromConstDoubleColumn(double value) {
- return new OTSColumn(new DoubleColumn(value), ColumnType.DOUBLE);
+ return new OTSColumn(new DoubleColumn(value));
}
public static OTSColumn fromConstBoolColumn(boolean value) {
- return new OTSColumn(new BoolColumn(value), ColumnType.BOOLEAN);
+ return new OTSColumn(new BoolColumn(value));
}
public static OTSColumn fromConstBytesColumn(byte[] value) {
- return new OTSColumn(new BytesColumn(value), ColumnType.BINARY);
+ return new OTSColumn(new BytesColumn(value));
}
public Column getValue() {
@@ -65,12 +63,25 @@ public class OTSColumn {
public OTSColumnType getColumnType() {
return columnType;
}
-
- public ColumnType getValueType() {
- return valueType;
- }
+
public String getName() {
return name;
}
-}
+
+ public ColumnType getValueType() {
+ return valueType;
+ }
+
+ public void setValueType(ColumnType valueType) {
+ this.valueType = valueType;
+ }
+
+ public Boolean getTimeseriesTag() {
+ return isTimeseriesTag;
+ }
+
+ public void setTimeseriesTag(Boolean timeseriesTag) {
+ isTimeseriesTag = timeseriesTag;
+ }
+}
\ No newline at end of file
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSConf.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSConf.java
index 8b109a39..cbfd8f6a 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSConf.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSConf.java
@@ -1,90 +1,245 @@
package com.alibaba.datax.plugin.reader.otsreader.model;
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.reader.otsreader.utils.Constant;
+import com.alibaba.datax.plugin.reader.otsreader.utils.Key;
+import com.alibaba.datax.plugin.reader.otsreader.utils.ParamChecker;
+import com.alicloud.openservices.tablestore.model.ColumnType;
+
import java.util.List;
-import com.aliyun.openservices.ots.model.PrimaryKeyValue;
-
public class OTSConf {
- private String endpoint= null;
+ private String endpoint = null;
private String accessId = null;
- private String accesskey = null;
+ private String accessKey = null;
private String instanceName = null;
private String tableName = null;
+ private OTSRange range = null;
+ private List column = null;
+ private OTSMode mode = null;
+
+ @Deprecated
+ private String metaMode = "";
+
+ private boolean newVersion = false;
+ /**
+ * 以下配置仅用于timeseries数据读取
+ */
+ private boolean isTimeseriesTable = false;
+ private String measurementName = null;
+ /**
+ * 以上配置仅用于timeseries数据读取
+ */
+ private OTSMultiVersionConf multi = null;
- private List rangeBegin = null;
- private List rangeEnd = null;
- private List rangeSplit = null;
-
- private List columns = null;
-
- private int retry;
- private int sleepInMilliSecond;
-
+ private int retry = Constant.ConfigDefaultValue.RETRY;
+ private int retryPauseInMillisecond = Constant.ConfigDefaultValue.RETRY_PAUSE_IN_MILLISECOND;
+ private int ioThreadCount = Constant.ConfigDefaultValue.IO_THREAD_COUNT;
+ private int maxConnectionCount = Constant.ConfigDefaultValue.MAX_CONNECTION_COUNT;
+ private int socketTimeoutInMillisecond = Constant.ConfigDefaultValue.SOCKET_TIMEOUT_IN_MILLISECOND;
+ private int connectTimeoutInMillisecond = Constant.ConfigDefaultValue.CONNECT_TIMEOUT_IN_MILLISECOND;
+
+ public int getIoThreadCount() {
+ return ioThreadCount;
+ }
+
+ public void setIoThreadCount(int ioThreadCount) {
+ this.ioThreadCount = ioThreadCount;
+ }
+
+ public int getMaxConnectCount() {
+ return maxConnectionCount;
+ }
+
+ public void setMaxConnectCount(int maxConnectCount) {
+ this.maxConnectionCount = maxConnectCount;
+ }
+
+ public int getSocketTimeoutInMillisecond() {
+ return socketTimeoutInMillisecond;
+ }
+
+ public void setSocketTimeoutInMillisecond(int socketTimeoutInMillisecond) {
+ this.socketTimeoutInMillisecond = socketTimeoutInMillisecond;
+ }
+
+ public int getConnectTimeoutInMillisecond() {
+ return connectTimeoutInMillisecond;
+ }
+
+ public void setConnectTimeoutInMillisecond(int connectTimeoutInMillisecond) {
+ this.connectTimeoutInMillisecond = connectTimeoutInMillisecond;
+ }
+
+ public int getRetry() {
+ return retry;
+ }
+
+ public void setRetry(int retry) {
+ this.retry = retry;
+ }
+
+ public int getRetryPauseInMillisecond() {
+ return retryPauseInMillisecond;
+ }
+
+ public void setRetryPauseInMillisecond(int sleepInMillisecond) {
+ this.retryPauseInMillisecond = sleepInMillisecond;
+ }
+
public String getEndpoint() {
return endpoint;
}
+
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}
+
public String getAccessId() {
return accessId;
}
+
public void setAccessId(String accessId) {
this.accessId = accessId;
}
- public String getAccesskey() {
- return accesskey;
+
+ public String getAccessKey() {
+ return accessKey;
}
- public void setAccesskey(String accesskey) {
- this.accesskey = accesskey;
+
+ public void setAccessKey(String accessKey) {
+ this.accessKey = accessKey;
}
+
public String getInstanceName() {
return instanceName;
}
+
public void setInstanceName(String instanceName) {
this.instanceName = instanceName;
}
+
public String getTableName() {
return tableName;
}
+
public void setTableName(String tableName) {
this.tableName = tableName;
}
- public List getColumns() {
- return columns;
+ public OTSRange getRange() {
+ return range;
}
- public void setColumns(List columns) {
- this.columns = columns;
+
+ public void setRange(OTSRange range) {
+ this.range = range;
}
- public int getRetry() {
- return retry;
+
+ public OTSMode getMode() {
+ return mode;
}
- public void setRetry(int retry) {
- this.retry = retry;
+
+ public void setMode(OTSMode mode) {
+ this.mode = mode;
}
- public int getSleepInMilliSecond() {
- return sleepInMilliSecond;
+
+ public OTSMultiVersionConf getMulti() {
+ return multi;
}
- public void setSleepInMilliSecond(int sleepInMilliSecond) {
- this.sleepInMilliSecond = sleepInMilliSecond;
+
+ public void setMulti(OTSMultiVersionConf multi) {
+ this.multi = multi;
}
- public List getRangeBegin() {
- return rangeBegin;
+
+ public List getColumn() {
+ return column;
}
- public void setRangeBegin(List rangeBegin) {
- this.rangeBegin = rangeBegin;
+
+ public void setColumn(List column) {
+ this.column = column;
}
- public List getRangeEnd() {
- return rangeEnd;
+
+ public boolean isNewVersion() {
+ return newVersion;
}
- public void setRangeEnd(List rangeEnd) {
- this.rangeEnd = rangeEnd;
+
+ public void setNewVersion(boolean newVersion) {
+ this.newVersion = newVersion;
}
- public List getRangeSplit() {
- return rangeSplit;
+
+ @Deprecated
+ public String getMetaMode() {
+ return metaMode;
}
- public void setRangeSplit(List rangeSplit) {
- this.rangeSplit = rangeSplit;
+
+ @Deprecated
+ public void setMetaMode(String metaMode) {
+ this.metaMode = metaMode;
+ }
+
+ public boolean isTimeseriesTable() {
+ return isTimeseriesTable;
+ }
+
+ public void setTimeseriesTable(boolean timeseriesTable) {
+ isTimeseriesTable = timeseriesTable;
+ }
+
+ public String getMeasurementName() {
+ return measurementName;
+ }
+
+ public void setMeasurementName(String measurementName) {
+ this.measurementName = measurementName;
+ }
+
+ public static OTSConf load(Configuration param) throws OTSCriticalException {
+ OTSConf c = new OTSConf();
+
+ // account
+ c.setEndpoint(ParamChecker.checkStringAndGet(param, Key.OTS_ENDPOINT, true));
+ c.setAccessId(ParamChecker.checkStringAndGet(param, Key.OTS_ACCESSID, true));
+ c.setAccessKey(ParamChecker.checkStringAndGet(param, Key.OTS_ACCESSKEY, true));
+ c.setInstanceName(ParamChecker.checkStringAndGet(param, Key.OTS_INSTANCE_NAME, true));
+ c.setTableName(ParamChecker.checkStringAndGet(param, Key.TABLE_NAME, true));
+
+ c.setRetry(param.getInt(Constant.ConfigKey.RETRY, Constant.ConfigDefaultValue.RETRY));
+ c.setRetryPauseInMillisecond(param.getInt(Constant.ConfigKey.RETRY_PAUSE_IN_MILLISECOND, Constant.ConfigDefaultValue.RETRY_PAUSE_IN_MILLISECOND));
+ c.setIoThreadCount(param.getInt(Constant.ConfigKey.IO_THREAD_COUNT, Constant.ConfigDefaultValue.IO_THREAD_COUNT));
+ c.setMaxConnectCount(param.getInt(Constant.ConfigKey.MAX_CONNECTION_COUNT, Constant.ConfigDefaultValue.MAX_CONNECTION_COUNT));
+ c.setSocketTimeoutInMillisecond(param.getInt(Constant.ConfigKey.SOCKET_TIMEOUTIN_MILLISECOND, Constant.ConfigDefaultValue.SOCKET_TIMEOUT_IN_MILLISECOND));
+ c.setConnectTimeoutInMillisecond(param.getInt(Constant.ConfigKey.CONNECT_TIMEOUT_IN_MILLISECOND, Constant.ConfigDefaultValue.CONNECT_TIMEOUT_IN_MILLISECOND));
+
+ // range
+ c.setRange(ParamChecker.checkRangeAndGet(param));
+
+ // mode 可选参数
+ c.setMode(ParamChecker.checkModeAndGet(param));
+ //isNewVersion 可选参数
+ c.setNewVersion(param.getBool(Key.NEW_VERSION, false));
+ // metaMode 旧版本配置
+ c.setMetaMode(param.getString(Key.META_MODE, ""));
+
+
+
+ // 读时序表配置项
+ c.setTimeseriesTable(param.getBool(Key.IS_TIMESERIES_TABLE, false));
+ // column
+ if(!c.isTimeseriesTable()){
+ //非时序表
+ c.setColumn(ParamChecker.checkOTSColumnAndGet(param, c.getMode()));
+ }
+ else{
+ // 时序表
+ c.setMeasurementName(param.getString(Key.MEASUREMENT_NAME, ""));
+ c.setColumn(ParamChecker.checkTimeseriesColumnAndGet(param));
+ ParamChecker.checkTimeseriesMode(c.getMode(), c.isNewVersion());
+ }
+
+ if (c.getMode() == OTSMode.MULTI_VERSION) {
+ c.setMulti(OTSMultiVersionConf.load(param));
+ }
+ return c;
}
}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSCriticalException.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSCriticalException.java
new file mode 100644
index 00000000..f02346bc
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSCriticalException.java
@@ -0,0 +1,24 @@
+package com.alibaba.datax.plugin.reader.otsreader.model;
+
+/**
+ * 插件错误异常，该异常主要用于描述插件的异常退出
+ * @author redchen
+ */
+public class OTSCriticalException extends Exception{
+
+ private static final long serialVersionUID = 5820460098894295722L;
+
+ public OTSCriticalException() {}
+
+ public OTSCriticalException(String message) {
+ super(message);
+ }
+
+ public OTSCriticalException(Throwable a) {
+ super(a);
+ }
+
+ public OTSCriticalException(String message, Throwable a) {
+ super(message, a);
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSErrorCode.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSErrorCode.java
new file mode 100644
index 00000000..0c537fce
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSErrorCode.java
@@ -0,0 +1,115 @@
+/**
+ * Copyright (C) Alibaba Cloud Computing
+ * All rights reserved.
+ *
+ * 版权所有 （C）阿里云计算有限公司
+ */
+
+package com.alibaba.datax.plugin.reader.otsreader.model;
+
+/**
+ * 表示来自开放结构化数据服务（Open Table Service，OTS）的错误代码。
+ *
+ */
+public class OTSErrorCode {
+ /**
+ * 用户身份验证失败。
+ */
+ public static final String AUTHORIZATION_FAILURE = "OTSAuthFailed";
+
+ /**
+ * 服务器内部错误。
+ */
+ public static final String INTERNAL_SERVER_ERROR = "OTSInternalServerError";
+
+ /**
+ * 参数错误。
+ */
+ public static final String INVALID_PARAMETER = "OTSParameterInvalid";
+
+ /**
+ * 整个请求过大。
+ */
+ public static final String REQUEST_TOO_LARGE = "OTSRequestBodyTooLarge";
+
+ /**
+ * 客户端请求超时。
+ */
+ public static final String REQUEST_TIMEOUT = "OTSRequestTimeout";
+
+ /**
+ * 用户的配额已经用满。
+ */
+ public static final String QUOTA_EXHAUSTED = "OTSQuotaExhausted";
+
+ /**
+ * 内部服务器发生failover，导致表的部分分区不可服务。
+ */
+ public static final String PARTITION_UNAVAILABLE = "OTSPartitionUnavailable";
+
+ /**
+ * 表刚被创建还无法立马提供服务。
+ */
+ public static final String TABLE_NOT_READY = "OTSTableNotReady";
+
+ /**
+ * 请求的表不存在。
+ */
+ public static final String OBJECT_NOT_EXIST = "OTSObjectNotExist";
+
+ /**
+ * 请求创建的表已经存在。
+ */
+ public static final String OBJECT_ALREADY_EXIST = "OTSObjectAlreadyExist";
+
+ /**
+ * 多个并发的请求写同一行数据，导致冲突。
+ */
+ public static final String ROW_OPEARTION_CONFLICT = "OTSRowOperationConflict";
+
+ /**
+ * 主键不匹配。
+ */
+ public static final String INVALID_PK = "OTSInvalidPK";
+
+ /**
+ * 读写能力调整过于频繁。
+ */
+ public static final String TOO_FREQUENT_RESERVED_THROUGHPUT_ADJUSTMENT = "OTSTooFrequentReservedThroughputAdjustment";
+
+ /**
+ * 该行总列数超出限制。
+ */
+ public static final String OUT_OF_COLUMN_COUNT_LIMIT = "OTSOutOfColumnCountLimit";
+
+ /**
+ * 该行所有列数据大小总和超出限制。
+ */
+ public static final String OUT_OF_ROW_SIZE_LIMIT = "OTSOutOfRowSizeLimit";
+
+ /**
+ * 剩余预留读写能力不足。
+ */
+ public static final String NOT_ENOUGH_CAPACITY_UNIT = "OTSNotEnoughCapacityUnit";
+
+ /**
+ * 预查条件检查失败。
+ */
+ public static final String CONDITION_CHECK_FAIL = "OTSConditionCheckFail";
+
+ /**
+ * 在OTS内部操作超时。
+ */
+ public static final String STORAGE_TIMEOUT = "OTSTimeout";
+
+ /**
+ * 在OTS内部有服务器不可访问。
+ */
+ public static final String SERVER_UNAVAILABLE = "OTSServerUnavailable";
+
+ /**
+ * OTS内部服务器繁忙。
+ */
+ public static final String SERVER_BUSY = "OTSServerBusy";
+
+}
\ No newline at end of file
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSMode.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSMode.java
new file mode 100644
index 00000000..88c6ee67
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSMode.java
@@ -0,0 +1,6 @@
+package com.alibaba.datax.plugin.reader.otsreader.model;
+
+public enum OTSMode {
+ NORMAL,
+ MULTI_VERSION
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSMultiVersionConf.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSMultiVersionConf.java
new file mode 100644
index 00000000..72a8e1b7
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSMultiVersionConf.java
@@ -0,0 +1,35 @@
+package com.alibaba.datax.plugin.reader.otsreader.model;
+
+import com.alibaba.datax.common.util.Configuration;
+import com.alibaba.datax.plugin.reader.otsreader.utils.Constant;
+import com.alibaba.datax.plugin.reader.otsreader.utils.ParamChecker;
+import com.alicloud.openservices.tablestore.model.TimeRange;
+
+public class OTSMultiVersionConf {
+
+ private TimeRange timeRange = null;
+ private int maxVersion = -1;
+
+ public TimeRange getTimeRange() {
+ return timeRange;
+ }
+
+ public void setTimeRange(TimeRange timeRange) {
+ this.timeRange = timeRange;
+ }
+
+ public int getMaxVersion() {
+ return maxVersion;
+ }
+
+ public void setMaxVersion(int maxVersion) {
+ this.maxVersion = maxVersion;
+ }
+
+ public static OTSMultiVersionConf load(Configuration param) throws OTSCriticalException {
+ OTSMultiVersionConf conf = new OTSMultiVersionConf();
+ conf.setTimeRange(ParamChecker.checkTimeRangeAndGet(param));
+ conf.setMaxVersion(param.getInt(Constant.ConfigKey.MAX_VERSION, Constant.ConfigDefaultValue.MAX_VERSION));
+ return conf;
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSPrimaryKeyColumn.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSPrimaryKeyColumn.java
index eaec50ce..44a37c0c 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSPrimaryKeyColumn.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSPrimaryKeyColumn.java
@@ -15,8 +15,41 @@ public class OTSPrimaryKeyColumn {
public PrimaryKeyType getType() {
return type;
}
+
+ public com.alicloud.openservices.tablestore.model.PrimaryKeyType getType(Boolean newVersion) {
+ com.alicloud.openservices.tablestore.model.PrimaryKeyType res = null;
+ switch (this.type){
+ case BINARY:
+ res = com.alicloud.openservices.tablestore.model.PrimaryKeyType.BINARY;
+ break;
+ case INTEGER:
+ res = com.alicloud.openservices.tablestore.model.PrimaryKeyType.INTEGER;
+ break;
+ case STRING:
+ default:
+ res = com.alicloud.openservices.tablestore.model.PrimaryKeyType.STRING;
+ break;
+ }
+ return res;
+ }
+
public void setType(PrimaryKeyType type) {
this.type = type;
}
+
+ public void setType(com.alicloud.openservices.tablestore.model.PrimaryKeyType type) {
+ switch (type){
+ case BINARY:
+ this.type = PrimaryKeyType.BINARY;
+ break;
+ case INTEGER:
+ this.type = PrimaryKeyType.INTEGER;
+ break;
+ case STRING:
+ default:
+ this.type = PrimaryKeyType.STRING;
+ break;
+ }
+ }
}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSRange.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSRange.java
index 8ebfcf7e..eb3095e6 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSRange.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSRange.java
@@ -1,29 +1,31 @@
package com.alibaba.datax.plugin.reader.otsreader.model;
-import com.aliyun.openservices.ots.model.RowPrimaryKey;
+import com.alicloud.openservices.tablestore.model.PrimaryKeyColumn;
+
+import java.util.List;
+
public class OTSRange {
+ private List begin = null;
+ private List end = null;
+ private List split = null;
- private RowPrimaryKey begin = null;
- private RowPrimaryKey end = null;
-
- public OTSRange() {}
-
- public OTSRange(RowPrimaryKey begin, RowPrimaryKey end) {
- this.begin = begin;
- this.end = end;
- }
-
- public RowPrimaryKey getBegin() {
+ public List getBegin() {
return begin;
}
- public void setBegin(RowPrimaryKey begin) {
+ public void setBegin(List begin) {
this.begin = begin;
}
- public RowPrimaryKey getEnd() {
+ public List getEnd() {
return end;
}
- public void setEnd(RowPrimaryKey end) {
+ public void setEnd(List end) {
this.end = end;
}
+ public List getSplit() {
+ return split;
+ }
+ public void setSplit(List split) {
+ this.split = split;
+ }
}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/Common.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/Common.java
index fb8c7feb..90065d5d 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/Common.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/Common.java
@@ -1,26 +1,85 @@
package com.alibaba.datax.plugin.reader.otsreader.utils;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSColumn;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSCriticalException;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSPrimaryKeyColumn;
+import com.alicloud.openservices.tablestore.model.*;
+import com.alicloud.openservices.tablestore.model.timeseries.ScanTimeseriesDataResponse;
+
+import java.lang.reflect.Field;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import com.alibaba.datax.common.element.BoolColumn;
-import com.alibaba.datax.common.element.BytesColumn;
-import com.alibaba.datax.common.element.DoubleColumn;
-import com.alibaba.datax.common.element.LongColumn;
-import com.alibaba.datax.common.element.Record;
-import com.alibaba.datax.common.element.StringColumn;
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSColumn;
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSPrimaryKeyColumn;
-import com.aliyun.openservices.ots.ClientException;
-import com.aliyun.openservices.ots.OTSException;
-import com.aliyun.openservices.ots.model.ColumnValue;
-import com.aliyun.openservices.ots.model.PrimaryKeyValue;
-import com.aliyun.openservices.ots.model.Row;
-import com.aliyun.openservices.ots.model.RowPrimaryKey;
-import com.aliyun.openservices.ots.model.TableMeta;
-
public class Common {
+ public static List toColumnToGet(List columns, TableMeta meta) {
+ Map pk = meta.getPrimaryKeyMap();
+ List names = new ArrayList();
+ for (OTSColumn c : columns) {
+ if (c.getColumnType() == OTSColumn.OTSColumnType.NORMAL && !pk.containsKey(c.getName())) {
+ names.add(c.getName());
+ }
+ }
+ return names;
+ }
+
+ public static List getPrimaryKeyNameList(TableMeta meta) {
+ List names = new ArrayList();
+ names.addAll(meta.getPrimaryKeyMap().keySet());
+ return names;
+ }
+
+ public static OTSPrimaryKeyColumn getPartitionKey(TableMeta meta) {
+ List keys = new ArrayList();
+ keys.addAll(meta.getPrimaryKeyMap().keySet());
+
+ String key = keys.get(0);
+
+ OTSPrimaryKeyColumn col = new OTSPrimaryKeyColumn();
+ col.setName(key);
+ col.setType(meta.getPrimaryKeyMap().get(key));
+ return col;
+ }
+
+ public static Direction getDirection(List begin, List end) throws OTSCriticalException {
+ int cmp = CompareHelper.comparePrimaryKeyColumnList(begin, end);
+ if (cmp < 0) {
+ return Direction.FORWARD;
+ } else if (cmp > 0) {
+ return Direction.BACKWARD;
+ } else {
+ throw new OTSCriticalException("Bug branch, the begin of range equals end of range.");
+ }
+ }
+
+ public static int compareRangeBeginAndEnd(TableMeta meta, List begin, List end) {
+ if (begin.size() != end.size()) {
+ throw new IllegalArgumentException("Input size of begin not equal size of end, begin size : " + begin.size() +
+ ", end size : " + end.size() + ".");
+ }
+
+ Map beginMap = new HashMap<>();
+ Map endMap = new HashMap<>();
+
+ for(PrimaryKeyColumn primaryKeyColumn : begin){
+ beginMap.put(primaryKeyColumn.getName(), primaryKeyColumn.getValue());
+ }
+ for(PrimaryKeyColumn primaryKeyColumn : end){
+ endMap.put(primaryKeyColumn.getName(), primaryKeyColumn.getValue());
+ }
+
+ for (String key : meta.getPrimaryKeyMap().keySet()) {
+ PrimaryKeyValue v1 = beginMap.get(key);
+ PrimaryKeyValue v2 = endMap.get(key);
+ int cmp = primaryKeyValueCmp(v1, v2);
+ if (cmp != 0) {
+ return cmp;
+ }
+ }
+ return 0;
+ }
+
public static int primaryKeyValueCmp(PrimaryKeyValue v1, PrimaryKeyValue v2) {
if (v1.getType() != null && v2.getType() != null) {
@@ -29,14 +88,14 @@ public class Common {
"Not same column type, column1:" + v1.getType() + ", column2:" + v2.getType());
}
switch (v1.getType()) {
- case INTEGER:
- Long l1 = Long.valueOf(v1.asLong());
- Long l2 = Long.valueOf(v2.asLong());
- return l1.compareTo(l2);
- case STRING:
- return v1.asString().compareTo(v2.asString());
- default:
- throw new IllegalArgumentException("Unsuporrt compare the type: " + v1.getType() + ".");
+ case INTEGER:
+ Long l1 = Long.valueOf(v1.asLong());
+ Long l2 = Long.valueOf(v2.asLong());
+ return l1.compareTo(l2);
+ case STRING:
+ return v1.asString().compareTo(v2.asString());
+ default:
+ throw new IllegalArgumentException("Unsuporrt compare the type: " + v1.getType() + ".");
}
} else {
if (v1 == v2) {
@@ -46,116 +105,31 @@ public class Common {
return -1;
} else if (v1 == PrimaryKeyValue.INF_MAX) {
return 1;
- }
+ }
if (v2 == PrimaryKeyValue.INF_MAX) {
return -1;
} else if (v2 == PrimaryKeyValue.INF_MIN) {
return 1;
- }
- }
- }
- return 0;
- }
-
- public static OTSPrimaryKeyColumn getPartitionKey(TableMeta meta) {
- List keys = new ArrayList();
- keys.addAll(meta.getPrimaryKey().keySet());
-
- String key = keys.get(0);
-
- OTSPrimaryKeyColumn col = new OTSPrimaryKeyColumn();
- col.setName(key);
- col.setType(meta.getPrimaryKey().get(key));
- return col;
- }
-
- public static List getPrimaryKeyNameList(TableMeta meta) {
- List names = new ArrayList();
- names.addAll(meta.getPrimaryKey().keySet());
- return names;
- }
-
- public static int compareRangeBeginAndEnd(TableMeta meta, RowPrimaryKey begin, RowPrimaryKey end) {
- if (begin.getPrimaryKey().size() != end.getPrimaryKey().size()) {
- throw new IllegalArgumentException("Input size of begin not equal size of end, begin size : " + begin.getPrimaryKey().size() +
- ", end size : " + end.getPrimaryKey().size() + ".");
- }
- for (String key : meta.getPrimaryKey().keySet()) {
- PrimaryKeyValue v1 = begin.getPrimaryKey().get(key);
- PrimaryKeyValue v2 = end.getPrimaryKey().get(key);
- int cmp = primaryKeyValueCmp(v1, v2);
- if (cmp != 0) {
- return cmp;
- }
- }
- return 0;
- }
-
- public static List getNormalColumnNameList(List columns) {
- List normalColumns = new ArrayList();
- for (OTSColumn col : columns) {
- if (col.getColumnType() == OTSColumn.OTSColumnType.NORMAL) {
- normalColumns.add(col.getName());
- }
- }
- return normalColumns;
- }
-
- public static Record parseRowToLine(Row row, List columns, Record line) {
- Map values = row.getColumns();
- for (OTSColumn col : columns) {
- if (col.getColumnType() == OTSColumn.OTSColumnType.CONST) {
- line.addColumn(col.getValue());
- } else {
- ColumnValue v = values.get(col.getName());
- if (v == null) {
- line.addColumn(new StringColumn(null));
- } else {
- switch(v.getType()) {
- case STRING: line.addColumn(new StringColumn(v.asString())); break;
- case INTEGER: line.addColumn(new LongColumn(v.asLong())); break;
- case DOUBLE: line.addColumn(new DoubleColumn(v.asDouble())); break;
- case BOOLEAN: line.addColumn(new BoolColumn(v.asBoolean())); break;
- case BINARY: line.addColumn(new BytesColumn(v.asBinary())); break;
- default:
- throw new IllegalArgumentException("Unsupported transform the type: " + col.getValue().getType() + ".");
- }
}
}
}
- return line;
+ return 0;
}
-
- public static String getDetailMessage(Exception exception) {
- if (exception instanceof OTSException) {
- OTSException e = (OTSException) exception;
- return "OTSException[ErrorCode:" + e.getErrorCode() + ", ErrorMessage:" + e.getMessage() + ", RequestId:" + e.getRequestId() + "]";
- } else if (exception instanceof ClientException) {
- ClientException e = (ClientException) exception;
- return "ClientException[ErrorCode:" + e.getErrorCode() + ", ErrorMessage:" + e.getMessage() + "]";
- } else if (exception instanceof IllegalArgumentException) {
- IllegalArgumentException e = (IllegalArgumentException) exception;
- return "IllegalArgumentException[ErrorMessage:" + e.getMessage() + "]";
- } else {
- return "Exception[ErrorMessage:" + exception.getMessage() + "]";
- }
- }
-
- public static long getDelaySendMillinSeconds(int hadRetryTimes, int initSleepInMilliSecond) {
- if (hadRetryTimes <= 0) {
- return 0;
- }
-
- int sleepTime = initSleepInMilliSecond;
- for (int i = 1; i < hadRetryTimes; i++) {
- sleepTime += sleepTime;
- if (sleepTime > 30000) {
- sleepTime = 30000;
+ public static void checkTableStoreSDKVersion() throws OTSCriticalException {
+ Field[] fields = ScanTimeseriesDataResponse.class.getFields();
+ String sdkVersion = null;
+ for (Field f : fields){
+ if (f.getName().equals("_VERSION_")){
+ sdkVersion = ScanTimeseriesDataResponse._VERSION_;
break;
- }
+ }
+ }
+ if (sdkVersion == null){
+ throw new OTSCriticalException("Check ots java SDK failed. Please check the version of tableStore maven dependency.");
+ }else if (Integer.parseInt(sdkVersion) < 20230111){
+ throw new OTSCriticalException("Check tableStore java SDK failed. The expected version number is greater than 20230111, actually version : " + sdkVersion + ".");
}
- return sleepTime;
}
}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/CommonOld.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/CommonOld.java
new file mode 100644
index 00000000..d5c565f4
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/CommonOld.java
@@ -0,0 +1,112 @@
+package com.alibaba.datax.plugin.reader.otsreader.utils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import com.alibaba.datax.common.element.BoolColumn;
+import com.alibaba.datax.common.element.BytesColumn;
+import com.alibaba.datax.common.element.DoubleColumn;
+import com.alibaba.datax.common.element.LongColumn;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.element.StringColumn;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSColumn;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSPrimaryKeyColumn;
+import com.aliyun.openservices.ots.ClientException;
+import com.aliyun.openservices.ots.OTSException;
+import com.aliyun.openservices.ots.model.ColumnValue;
+import com.aliyun.openservices.ots.model.PrimaryKeyValue;
+import com.aliyun.openservices.ots.model.Row;
+import com.aliyun.openservices.ots.model.RowPrimaryKey;
+import com.aliyun.openservices.ots.model.TableMeta;
+
+public class CommonOld {
+ public static int primaryKeyValueCmp(PrimaryKeyValue v1, PrimaryKeyValue v2) {
+ if (v1.getType() != null && v2.getType() != null) {
+ if (v1.getType() != v2.getType()) {
+ throw new IllegalArgumentException(
+ "Not same column type, column1:" + v1.getType() + ", column2:" + v2.getType());
+ }
+ switch (v1.getType()) {
+ case INTEGER:
+ Long l1 = Long.valueOf(v1.asLong());
+ Long l2 = Long.valueOf(v2.asLong());
+ return l1.compareTo(l2);
+ case STRING:
+ return v1.asString().compareTo(v2.asString());
+ default:
+ throw new IllegalArgumentException("Unsuporrt compare the type: " + v1.getType() + ".");
+ }
+ } else {
+ if (v1 == v2) {
+ return 0;
+ } else {
+ if (v1 == PrimaryKeyValue.INF_MIN) {
+ return -1;
+ } else if (v1 == PrimaryKeyValue.INF_MAX) {
+ return 1;
+ }
+
+ if (v2 == PrimaryKeyValue.INF_MAX) {
+ return -1;
+ } else if (v2 == PrimaryKeyValue.INF_MIN) {
+ return 1;
+ }
+ }
+ }
+ return 0;
+ }
+
+
+ public static List getNormalColumnNameList(List columns) {
+ List normalColumns = new ArrayList();
+ for (OTSColumn col : columns) {
+ if (col.getColumnType() == OTSColumn.OTSColumnType.NORMAL) {
+ normalColumns.add(col.getName());
+ }
+ }
+ return normalColumns;
+ }
+
+ public static Record parseRowToLine(Row row, List columns, Record line) {
+ Map values = row.getColumns();
+ for (OTSColumn col : columns) {
+ if (col.getColumnType() == OTSColumn.OTSColumnType.CONST) {
+ line.addColumn(col.getValue());
+ } else {
+ ColumnValue v = values.get(col.getName());
+ if (v == null) {
+ line.addColumn(new StringColumn(null));
+ } else {
+ switch(v.getType()) {
+ case STRING: line.addColumn(new StringColumn(v.asString())); break;
+ case INTEGER: line.addColumn(new LongColumn(v.asLong())); break;
+ case DOUBLE: line.addColumn(new DoubleColumn(v.asDouble())); break;
+ case BOOLEAN: line.addColumn(new BoolColumn(v.asBoolean())); break;
+ case BINARY: line.addColumn(new BytesColumn(v.asBinary())); break;
+ default:
+ throw new IllegalArgumentException("Unsuporrt tranform the type: " + col.getValue().getType() + ".");
+ }
+ }
+ }
+ }
+ return line;
+ }
+
+ public static long getDelaySendMillinSeconds(int hadRetryTimes, int initSleepInMilliSecond) {
+
+ if (hadRetryTimes <= 0) {
+ return 0;
+ }
+
+ int sleepTime = initSleepInMilliSecond;
+ for (int i = 1; i < hadRetryTimes; i++) {
+ sleepTime += sleepTime;
+ if (sleepTime > 30000) {
+ sleepTime = 30000;
+ break;
+ }
+ }
+ return sleepTime;
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/CompareHelper.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/CompareHelper.java
new file mode 100644
index 00000000..19e06421
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/CompareHelper.java
@@ -0,0 +1,37 @@
+package com.alibaba.datax.plugin.reader.otsreader.utils;
+
+import com.alicloud.openservices.tablestore.model.PrimaryKeyColumn;
+
+import java.util.List;
+
+
+public class CompareHelper {
+ /**
+ * 比较PrimaryKeyColumn List的大小
+ * 返回
+ * -1 表示before小于after
+ * 0 表示before等于after
+ * 1 表示before大于after
+ *
+ * @param before
+ * @param after
+ * @return
+ */
+ public static int comparePrimaryKeyColumnList(List before, List after) {
+ int size = before.size() < after.size() ? before.size() : after.size();
+
+ for (int i = 0; i < size; i++) {
+ int cmp = before.get(i).compareTo(after.get(i));
+ if (cmp != 0) {
+ return cmp;
+ }
+ }
+
+ if (before.size() < after.size() ) {
+ return -1;
+ } else if (before.size() > after.size() ) {
+ return 1;
+ }
+ return 0;
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/Constant.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/Constant.java
new file mode 100644
index 00000000..90273bfb
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/Constant.java
@@ -0,0 +1,92 @@
+package com.alibaba.datax.plugin.reader.otsreader.utils;
+
+public class Constant {
+ /**
+ * Json中的Key名字定义
+ */
+public class ConfigKey {
+ public static final String CONF = "conf";
+ public static final String RANGE = "range";
+ public static final String META = "meta";
+ public static final String SPLIT_INFO = "splitInfo";
+
+ public static final String TIME_RANGE = "timeRange";
+ public static final String MAX_VERSION = "maxVersion";
+
+ public static final String RETRY = "maxRetryTime";
+ public static final String RETRY_PAUSE_IN_MILLISECOND = "retryPauseInMillisecond";
+ public static final String IO_THREAD_COUNT = "ioThreadCount";
+ public static final String MAX_CONNECTION_COUNT = "maxConnectionCount";
+ public static final String SOCKET_TIMEOUTIN_MILLISECOND = "socketTimeoutInMillisecond";
+ public static final String CONNECT_TIMEOUT_IN_MILLISECOND = "connectTimeoutInMillisecond";
+
+ public class Range {
+ public static final String BEGIN = "begin";
+ public static final String END = "end";
+ public static final String SPLIT = "split";
+ };
+
+ public class PrimaryKeyColumn {
+ public static final String TYPE = "type";
+ public static final String VALUE = "value";
+ };
+
+ public class TimeseriesPKColumn {
+ public static final String MEASUREMENT_NAME = "_m_name";
+ public static final String DATA_SOURCE = "_data_source";
+ public static final String TAGS = "_tags";
+ public static final String TIME = "_time";
+ }
+
+ public class Column {
+ public static final String NAME = "name";
+ public static final String TYPE = "type";
+ public static final String VALUE = "value";
+ public static final String IS_TAG = "is_timeseries_tag";
+ };
+
+ public class TimeRange {
+ public static final String BEGIN = "begin";
+ public static final String END = "end";
+ }
+ };
+
+ /**
+ * 定义的配置文件中value type中可取的值
+ */
+ public class ValueType {
+ public static final String INF_MIN = "INF_MIN";
+ public static final String INF_MAX = "INF_MAX";
+ public static final String STRING = "string";
+ public static final String INTEGER = "int";
+ public static final String BINARY = "binary";
+ public static final String DOUBLE = "double";
+ public static final String BOOLEAN = "bool";
+ };
+
+ /**
+ * 全局默认常量定义
+ */
+ public class ConfigDefaultValue {
+ public static final int RETRY = 18;
+ public static final int RETRY_PAUSE_IN_MILLISECOND = 100;
+ public static final int IO_THREAD_COUNT = 1;
+ public static final int MAX_CONNECTION_COUNT = 1;
+ public static final int SOCKET_TIMEOUT_IN_MILLISECOND = 10000;
+ public static final int CONNECT_TIMEOUT_IN_MILLISECOND = 10000;
+
+ public static final int MAX_VERSION = Integer.MAX_VALUE;
+
+ public static final String DEFAULT_NAME = "DEFAULT_NAME";
+
+ public class Mode {
+ public static final String NORMAL = "normal";
+ public static final String MULTI_VERSION = "multiVersion";
+ }
+
+ public class TimeRange {
+ public static final long MIN = 0;
+ public static final long MAX = Long.MAX_VALUE;
+ }
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/GsonParser.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/GsonParser.java
index a82f3350..205f536d 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/GsonParser.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/GsonParser.java
@@ -1,23 +1,26 @@
package com.alibaba.datax.plugin.reader.otsreader.utils;
-import com.alibaba.datax.plugin.reader.otsreader.adaptor.OTSColumnAdaptor;
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.plugin.reader.otsreader.adaptor.ColumnAdaptor;
import com.alibaba.datax.plugin.reader.otsreader.adaptor.PrimaryKeyValueAdaptor;
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSColumn;
import com.alibaba.datax.plugin.reader.otsreader.model.OTSConf;
import com.alibaba.datax.plugin.reader.otsreader.model.OTSRange;
+import com.alicloud.openservices.tablestore.model.PrimaryKeyValue;
+import com.alicloud.openservices.tablestore.model.TableMeta;
+import com.alicloud.openservices.tablestore.model.timeseries.TimeseriesScanSplitInfo;
import com.aliyun.openservices.ots.model.Direction;
-import com.aliyun.openservices.ots.model.PrimaryKeyValue;
import com.aliyun.openservices.ots.model.RowPrimaryKey;
-import com.aliyun.openservices.ots.model.TableMeta;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import java.util.Map;
+
public class GsonParser {
private static Gson gsonBuilder() {
return new GsonBuilder()
- .registerTypeAdapter(OTSColumn.class, new OTSColumnAdaptor())
.registerTypeAdapter(PrimaryKeyValue.class, new PrimaryKeyValueAdaptor())
+ .registerTypeAdapter(Column.class, new ColumnAdaptor())
.create();
}
@@ -40,24 +43,39 @@ public class GsonParser {
Gson g = gsonBuilder();
return g.fromJson(jsonStr, OTSConf.class);
}
-
- public static String directionToJson (Direction direction) {
+
+ public static String metaToJson (TableMeta meta) {
Gson g = gsonBuilder();
- return g.toJson(direction);
+ return g.toJson(meta);
+ }
+
+ public static TableMeta jsonToMeta (String jsonStr) {
+ Gson g = gsonBuilder();
+ return g.fromJson(jsonStr, TableMeta.class);
+ }
+
+ public static String timeseriesScanSplitInfoToString(TimeseriesScanSplitInfo timeseriesScanSplitInfo){
+ Gson g = gsonBuilder();
+ return g.toJson(timeseriesScanSplitInfo);
+ }
+
+ public static TimeseriesScanSplitInfo stringToTimeseriesScanSplitInfo(String jsonStr){
+ Gson g = gsonBuilder();
+ return g.fromJson(jsonStr, TimeseriesScanSplitInfo.class);
}
public static Direction jsonToDirection (String jsonStr) {
Gson g = gsonBuilder();
return g.fromJson(jsonStr, Direction.class);
}
-
- public static String metaToJson (TableMeta meta) {
- Gson g = gsonBuilder();
- return g.toJson(meta);
- }
-
+
public static String rowPrimaryKeyToJson (RowPrimaryKey row) {
Gson g = gsonBuilder();
return g.toJson(row);
}
+
+ public static String mapToJson (Map map) {
+ Gson g = gsonBuilder();
+ return g.toJson(map);
+ }
}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/Key.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/Key.java
similarity index 81%
rename from otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/Key.java
rename to otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/Key.java
index da6d4a5f..6628e4d3 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/Key.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/Key.java
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.alibaba.datax.plugin.reader.otsreader;
+package com.alibaba.datax.plugin.reader.otsreader.utils;
public final class Key {
/* ots account configuration */
@@ -46,5 +46,13 @@ public final class Key {
public final static String RANGE_END = "end";
public final static String RANGE_SPLIT = "split";
+
+ public final static String META_MODE = "metaMode";
+
+ public final static String MODE = "mode";
+ public final static String NEW_VERSION = "newVersion";
+
+ public final static String IS_TIMESERIES_TABLE = "isTimeseriesTable";
+ public final static String MEASUREMENT_NAME = "measurementName";
}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/OtsHelper.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/OtsHelper.java
new file mode 100644
index 00000000..060507b6
--- /dev/null
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/OtsHelper.java
@@ -0,0 +1,82 @@
+package com.alibaba.datax.plugin.reader.otsreader.utils;
+
+import com.alibaba.datax.plugin.reader.otsreader.callable.GetRangeCallable;
+import com.alibaba.datax.plugin.reader.otsreader.callable.GetTableMetaCallable;
+import com.alibaba.datax.plugin.reader.otsreader.callable.GetTimeseriesSplitCallable;
+import com.alibaba.datax.plugin.reader.otsreader.callable.ScanTimeseriesDataCallable;
+import com.alibaba.datax.plugin.reader.otsreader.model.DefaultNoRetry;
+import com.alibaba.datax.plugin.reader.otsreader.model.OTSConf;
+import com.alicloud.openservices.tablestore.ClientConfiguration;
+import com.alicloud.openservices.tablestore.SyncClient;
+import com.alicloud.openservices.tablestore.SyncClientInterface;
+import com.alicloud.openservices.tablestore.core.utils.Pair;
+import com.alicloud.openservices.tablestore.model.ColumnType;
+import com.alicloud.openservices.tablestore.model.GetRangeResponse;
+import com.alicloud.openservices.tablestore.model.RangeRowQueryCriteria;
+import com.alicloud.openservices.tablestore.model.TableMeta;
+import com.alicloud.openservices.tablestore.model.timeseries.ScanTimeseriesDataRequest;
+import com.alicloud.openservices.tablestore.model.timeseries.ScanTimeseriesDataResponse;
+import com.alicloud.openservices.tablestore.model.timeseries.TimeseriesScanSplitInfo;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class OtsHelper {
+
+ public static SyncClientInterface getOTSInstance(OTSConf conf) {
+ ClientConfiguration clientConfigure = new ClientConfiguration();
+ clientConfigure.setIoThreadCount(conf.getIoThreadCount());
+ clientConfigure.setMaxConnections(conf.getMaxConnectCount());
+ clientConfigure.setSocketTimeoutInMillisecond(conf.getSocketTimeoutInMillisecond());
+ clientConfigure.setConnectionTimeoutInMillisecond(conf.getConnectTimeoutInMillisecond());
+ clientConfigure.setRetryStrategy(new DefaultNoRetry());
+
+ SyncClient ots = new SyncClient(
+ conf.getEndpoint(),
+ conf.getAccessId(),
+ conf.getAccessKey(),
+ conf.getInstanceName(),
+ clientConfigure);
+
+
+ Map extraHeaders = new HashMap();
+ extraHeaders.put("x-ots-sdk-type", "public");
+ extraHeaders.put("x-ots-request-source", "datax-otsreader");
+ ots.setExtraHeaders(extraHeaders);
+
+ return ots;
+ }
+
+ public static TableMeta getTableMeta(SyncClientInterface ots, String tableName, int retry, int sleepInMillisecond) throws Exception {
+ return RetryHelper.executeWithRetry(
+ new GetTableMetaCallable(ots, tableName),
+ retry,
+ sleepInMillisecond
+ );
+ }
+
+ public static GetRangeResponse getRange(SyncClientInterface ots, RangeRowQueryCriteria rangeRowQueryCriteria, int retry, int sleepInMillisecond) throws Exception {
+ return RetryHelper.executeWithRetry(
+ new GetRangeCallable(ots, rangeRowQueryCriteria),
+ retry,
+ sleepInMillisecond
+ );
+ }
+
+ public static List splitTimeseriesScan(SyncClientInterface ots, String tableName, String measurementName, int splitCountHint, int retry, int sleepInMillisecond) throws Exception {
+ return RetryHelper.executeWithRetry(
+ new GetTimeseriesSplitCallable(ots, tableName, measurementName, splitCountHint),
+ retry,
+ sleepInMillisecond
+ );
+ }
+
+ public static ScanTimeseriesDataResponse scanTimeseriesData(SyncClientInterface ots, ScanTimeseriesDataRequest scanTimeseriesDataRequest, int retry, int sleepInMillisecond) throws Exception {
+ return RetryHelper.executeWithRetry(
+ new ScanTimeseriesDataCallable(ots, scanTimeseriesDataRequest),
+ retry,
+ sleepInMillisecond
+ );
+ }
+}
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderError.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/OtsReaderError.java
similarity index 76%
rename from otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderError.java
rename to otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/OtsReaderError.java
index 05a13c1a..b578dcde 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderError.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/OtsReaderError.java
@@ -1,4 +1,4 @@
-package com.alibaba.datax.plugin.reader.otsreader;
+package com.alibaba.datax.plugin.reader.otsreader.utils;
import com.alibaba.datax.common.spi.ErrorCode;
@@ -14,10 +14,10 @@ public class OtsReaderError implements ErrorCode {
public final static OtsReaderError ERROR = new OtsReaderError(
"OtsReaderError",
- "该错误表示插件的内部错误，表示系统没有处理到的异常");
+ "This error represents an internal error of the otsreader plugin, which indicates that the system is not processed.");
public final static OtsReaderError INVALID_PARAM = new OtsReaderError(
"OtsReaderInvalidParameter",
- "该错误表示参数错误，表示用户输入了错误的参数格式等");
+ "This error represents a parameter error, indicating that the user entered the wrong parameter format.");
public OtsReaderError (String code) {
this.code = code;
diff --git a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/ParamChecker.java b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/ParamChecker.java
index fbcdc972..b2139fc1 100644
--- a/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/ParamChecker.java
+++ b/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/ParamChecker.java
@@ -1,162 +1,40 @@
package com.alibaba.datax.plugin.reader.otsreader.utils;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
+import com.alibaba.datax.common.element.Column;
import com.alibaba.datax.common.util.Configuration;
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSPrimaryKeyColumn;
-import com.alibaba.datax.plugin.reader.otsreader.model.OTSRange;
-import com.aliyun.openservices.ots.model.Direction;
-import com.aliyun.openservices.ots.model.PrimaryKeyType;
-import com.aliyun.openservices.ots.model.PrimaryKeyValue;
-import com.aliyun.openservices.ots.model.RowPrimaryKey;
-import com.aliyun.openservices.ots.model.TableMeta;
+import com.alibaba.datax.plugin.reader.otsreader.model.*;
+import com.alicloud.openservices.tablestore.model.*;
+
+import java.util.*;
public class ParamChecker {
- private static void throwNotExistException(String key) {
- throw new IllegalArgumentException("The param '" + key + "' is not exist.");
+ private static void throwNotExistException() {
+ throw new IllegalArgumentException("missing the key.");
}
- private static void throwStringLengthZeroException(String key) {
- throw new IllegalArgumentException("The param length of '" + key + "' is zero.");
+ private static void throwStringLengthZeroException() {
+ throw new IllegalArgumentException("input the key is empty string.");
}
- private static void throwEmptyException(String key) {
- throw new IllegalArgumentException("The param '" + key + "' is empty.");
- }
-
- private static void throwNotListException(String key) {
- throw new IllegalArgumentException("The param '" + key + "' is not a json array.");
- }
-
- private static void throwNotMapException(String key) {
- throw new IllegalArgumentException("The param '" + key + "' is not a json map.");
- }
-
- public static String checkStringAndGet(Configuration param, String key) {
- String value = param.getString(key);
- if (null == value) {
- throwNotExistException(key);
- } else if (value.length() == 0) {
- throwStringLengthZeroException(key);
- }
- return value;
- }
-
- public static List