From c5c41de1b18ba635a3bf23f62cf770846076acf0 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 2 Mar 2022 17:21:50 +0800 Subject: [PATCH] change --- .../datax/plugin/reader/TDengineReader.java | 18 +++--- .../resources/incremental_sync/clean_env.sh | 9 --- .../incremental_sync/dm2t-update.json | 63 ------------------- .../resources/incremental_sync/dm2t_sync.sh | 57 ----------------- .../resources/incremental_sync/t2dm-jni.json | 50 --------------- .../incremental_sync/t2dm-restful.json | 50 --------------- .../test/resources/incremental_sync/upload.sh | 6 -- 7 files changed, 10 insertions(+), 243 deletions(-) delete mode 100755 tdenginewriter/src/test/resources/incremental_sync/clean_env.sh delete mode 100644 tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json delete mode 100755 tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh delete mode 100644 tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json delete mode 100644 tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json delete mode 100755 tdenginewriter/src/test/resources/incremental_sync/upload.sh diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java index aea75f83..0cf214c1 100644 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -127,6 +127,15 @@ public class TDengineReader extends Reader { private String startTime; private String endTime; + static { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + @Override public void init() { this.readerSliceConfig = super.getPluginJobConf(); @@ -237,13 +246,6 @@ public class TDengineReader extends Reader { } } - static { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } + } diff --git a/tdenginewriter/src/test/resources/incremental_sync/clean_env.sh b/tdenginewriter/src/test/resources/incremental_sync/clean_env.sh deleted file mode 100755 index f3dca7c1..00000000 --- a/tdenginewriter/src/test/resources/incremental_sync/clean_env.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -datax_home_dir=$(dirname $(readlink -f "$0")) - -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'drop table if exists db2.stb2;' 192.168.1.93:6041/rest/sql -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create table if not exists db2.stb2 (`ts` TIMESTAMP,`f2` SMALLINT,`f4` BIGINT,`f5` FLOAT,`f6` DOUBLE,`f7` DOUBLE,`f8` BOOL,`f9` NCHAR(100),`f10` NCHAR(200)) TAGS (`f1` TINYINT,`f3` INT);' 192.168.1.93:6041/rest/sql - -rm -f ${datax_home_dir}/log/* -rm -f ${datax_home_dir}/job/*.csv \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json b/tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json deleted file mode 100644 index d9285b23..00000000 --- a/tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "rdbmsreader", - "parameter": { - "username": "TESTUSER", - "password": "test123456", - "connection": [ - { - "querySql": [ - "select concat(concat(concat('t', f1), '_'),f3) as tbname,* from stb1" - ], - "jdbcUrl": [ - "jdbc:dm://192.168.0.72:5236" - ] - } - ], - "where": "1=1", - "fetchSize": 1024 - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "username": "root", - "password": "taosdata", - "column": [ - "tbname", - "ts", - "f1", - "f2", - "f3", - "f4", - "f5", - "f6", - "f7", - "f8", - "f9", - "f10" - ], - "connection": [ - { - "table": [ - "stb2" - ], - "jdbcUrl": "jdbc:TAOS-RS://192.168.1.93:6041/db2" - } - ], - "batchSize": 1000, - "ignoreTagsUnmatched": true - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh b/tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh deleted file mode 100755 index 426c6233..00000000 --- a/tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -set -e -#set -x - -datax_home_dir=$(dirname $(readlink -f "$0")) -table_name="stb1" -update_key="ts" - -while getopts "hd:t:" arg; do - case $arg in - d) - datax_home_dir=$(echo $OPTARG) - ;; - v) - table_name=$(echo $OPTARG) - ;; - h) - echo "Usage: $(basename $0) -d [datax_home_dir] -t [table_name] -k [update_key]" - echo " -h help" - exit 0 - ;; - ?) #unknow option - echo "unkonw argument" - exit 1 - ;; - esac -done - -if [[ -e ${datax_home_dir}/job/${table_name}.csv ]]; then - MAX_TIME=$(cat ${datax_home_dir}/job/${table_name}.csv) -else - MAX_TIME="null" -fi -current_datetime=$(date +"%Y-%m-%d %H:%M:%S") -current_timestamp=$(date +%s) - -if [ "$MAX_TIME" != "null" ]; then - WHERE="${update_key} >= '$MAX_TIME' and ${update_key} < '$current_datetime'" - sed "s/1=1/$WHERE/g" ${datax_home_dir}/job/dm2t-update.json >${datax_home_dir}/job/dm2t_${current_timestamp}.json - echo "incremental data synchronization, from '${MAX_TIME}' to '${current_datetime}'" - python ${datax_home_dir}/bin/datax.py ${datax_home_dir}/job/dm2t_${current_timestamp}.json 1> /dev/null 2>&1 -else - echo "full data synchronization, to '${current_datetime}'" - python ${datax_home_dir}/bin/datax.py ${datax_home_dir}/job/dm2t-update.json 1> /dev/null 2>&1 -fi - -if [[ $? -ne 0 ]]; then - echo "datax migration job falied" -else - echo ${current_datetime} >$datax_home_dir/job/${table_name}.csv - echo "datax migration job success" -fi - -rm -rf ${datax_home_dir}/job/dm2t_${current_timestamp}.json - -#while true; do ./dm2t_sync.sh; sleep 5s; done \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json b/tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json deleted file mode 100644 index 341f6293..00000000 --- a/tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "tdenginereader", - "parameter": { - "username": "root", - "password": "taosdata", - "column": [ - "*" - ], - "connection": [ - { - "table": [ - "stb1" - ], - "jdbcUrl": "jdbc:TAOS://192.168.56.105:6030/db1" - } - ] - } - }, - "writer": { - "name": "rdbmswriter", - "parameter": { - "connection": [ - { - "table": [ - "stb2" - ], - "jdbcUrl": "jdbc:dm://192.168.0.72:5236" - } - ], - "username": "TESTUSER", - "password": "test123456", - "table": "stb2", - "column": [ - "*" - ] - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json b/tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json deleted file mode 100644 index b2cf91e2..00000000 --- a/tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "tdenginereader", - "parameter": { - "username": "root", - "password": "taosdata", - "column": [ - "*" - ], - "connection": [ - { - "table": [ - "stb1" - ], - "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db1" - } - ] - } - }, - "writer": { - "name": "rdbmswriter", - "parameter": { - "connection": [ - { - "table": [ - "stb2" - ], - "jdbcUrl": "jdbc:dm://192.168.0.72:5236" - } - ], - "username": "TESTUSER", - "password": "test123456", - "table": "stb2", - "column": [ - "*" - ] - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/upload.sh b/tdenginewriter/src/test/resources/incremental_sync/upload.sh deleted file mode 100755 index 9b2a35e3..00000000 --- a/tdenginewriter/src/test/resources/incremental_sync/upload.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -scp dm2t-update.json root@192.168.56.105:/root/workspace/tmp/datax/job -scp dm2t_sync.sh root@192.168.56.105:/root/workspace/tmp/datax -scp clean_env.sh root@192.168.56.105:/root/workspace/tmp/datax -scp \ No newline at end of file