github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 06:21:11 +08:00
Code Issues Projects Releases Wiki Activity

Revert "SQOOP-3289: Add .travis.yml"

Browse Source 
This reverts commit 83a18e1951.

Reverting because of accidental inclusion of .cache file.
This commit is contained in:
Fero Szabo 2018-11-23 15:05:05 +01:00
parent 83a18e1951
commit cbc39c3bfa
37 changed files with 395 additions and 604 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
.travis.yml
View File

@ -1,82 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
dist: trusty
sudo: true
language: java
jdk:
- oraclejdk8
services:
- docker
env:
- TEST_TASK=test
- TEST_TASK=oracleXeTest
- TEST_TASK=sqlServerTest
- TEST_TASK=mySqlTest
- TEST_TASK=postgreSqlTest
- TEST_TASK=db2Test
- TEST_TASK=cubridTest
- TEST_TASK=mainFrameTest
before_install:
- if [ "$TEST_TASK" = "postgreSqlTest" ];
then
./src/scripts/thirdpartytest/start-thirdpartytest-db-containers.sh postgresql;
sleep 60;
elif [ "$TEST_TASK" = "mySqlTest" ];
then
./src/scripts/thirdpartytest/start-thirdpartytest-db-containers.sh mysql;
sleep 60;
elif [ "$TEST_TASK" = "cubridTest" ];
then
./src/scripts/thirdpartytest/start-thirdpartytest-db-containers.sh cubrid;
sleep 60;
elif [ "$TEST_TASK" = "sqlServerTest" ];
then
./src/scripts/thirdpartytest/start-thirdpartytest-db-containers.sh mssql;
sleep 60;
elif [ "$TEST_TASK" = "db2Test" ];
then
./src/scripts/thirdpartytest/start-thirdpartytest-db-containers.sh db2;
sleep 60;
elif [ "$TEST_TASK" = "mainFrameTest" ];
then
./src/scripts/thirdpartytest/start-thirdpartytest-db-containers.sh mainframe;
sleep 60;
elif [ "$TEST_TASK" = "oracleXeTest" ];
then
if [ -n "$ORACLE_DRIVER_URL" ];
then
mkdir -p drivers;
wget -q $ORACLE_DRIVER_URL --output-document ./drivers/oracledriver.jar;
./src/scripts/thirdpartytest/start-thirdpartytest-db-containers.sh oracle;
sleep 90;
fi
fi
script:
- if [ "$TEST_TASK" != "oracleXeTest" ] || [ -n "$ORACLE_DRIVER_URL" ];
then
./gradlew -Dsqoop.thirdparty.lib.dir=./drivers $TEST_TASK;
else
echo 'Skipping Oracle XE test execution because ORACLE_DRIVER_URL is not set.';
fi
before_cache:
- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
- rm -fr $HOME/.gradle/caches/*/plugin-resolution/
cache:
directories:
- $HOME/.gradle/caches/
- $HOME/.gradle/wrapper/

246
COMPILING.txt
View File

@ -87,22 +87,22 @@ The following table shows the currently supported test categories and their hier
.Available test categories .Available test categories
[width="40%",frame="topbot",options="header"] [width="40%",frame="topbot",options="header"]
|=============================================================== |======================
|Category |Subcategories | |Category |Subcategory
.3+|+SqoopTest+ |+UnitTest+ | .3+|+SqoopTest+ |+UnitTest+
|+IntegrationTest+ | |+IntegrationTest+
|+ManualTest+ | |+ManualTest+
.9+|+ThirdPartyTest+ |+CubridTest+ | .9+|+ThirdPartyTest+ |+CubridTest+
|+Db2Test+ | |+Db2Test+
|+MainFrameTest+ | |+MainFrameTest+
|+MysqlTest+ | |+MysqlTest+
|+NetezzaTest+ | |+NetezzaTest+
|+OracleTest+ |+OracleEeTest+ |+OracleTest+
|+PostgresqlTest+ | |+PostgresqlTest+
|+SqlServerTest+ | |+SqlServerTest+
|+S3Test+ | |+S3Test+
|+KerberizedTest+ | | |+KerberizedTest+ |
|=============================================================== |======================
==== SqoopTest ==== SqoopTest
A general category including UnitTest, IntegrationTest and ManualTest. A general category including UnitTest, IntegrationTest and ManualTest.
@ -143,14 +143,6 @@ corresponding marker interface to it.
* +integrationTest+: Runs integration tests which do not need a docker container or an external database/service. * +integrationTest+: Runs integration tests which do not need a docker container or an external database/service.
* +kerberizedTest+: Runs kerberized tests. * +kerberizedTest+: Runs kerberized tests.
* +thirdPartyTest+: Runs third-party tests. For more information see the 'Third party tests' section. * +thirdPartyTest+: Runs third-party tests. For more information see the 'Third party tests' section.
* +postgreSqlTest+: Runs PostgreSql third-party tests.
* +mySqlTest+: Runs MySql third-party tests.
* +cubridTest+: Runs Cubrid third-party tests.
* +db2Test+: Runs DB2 third-party tests.
* +oracleTest+: Runs Oracle third-party tests.
* +oracleXeTest+: Run Oracle third party tests which can be run with Oracle Express Edition too.
* +oracleEeTest+: Run Oracle third party tests which require Oracle Enterprise Edition.
* +sqlServerTest+: Runs SqlServer third-party tests.
* +test+: Runs tests that do not need external JDBC driver and/or a docker container. * +test+: Runs tests that do not need external JDBC driver and/or a docker container.
This the same as running unitTest, integrationTest and kerberizedTest. This the same as running unitTest, integrationTest and kerberizedTest.
* +s3Test+: Runs S3 tests. For more information see the 'Amazon S3 tests' section. * +s3Test+: Runs S3 tests. For more information see the 'Amazon S3 tests' section.
@ -180,41 +172,36 @@ JDBC. Instructions for configuring the MySQL database are in MySQLAuthTest
and DirectMySQLTest. and DirectMySQLTest.
Use the following system properties to configure connection to the MySQL host used for testing: Use the following system properties to configure connection to the MySQL host used for testing:
+sqoop.test.mysql.connectstring.host_url+, +sqoop.test.mysql.databasename+, +sqoop.test.mysql.username+ and sqoop.test.mysql.connectstring.host_url, sqoop.test.mysql.databasename, sqoop.test.mysql.username and
+sqoop.test.mysql.password+. sqoop.test.mysql.password.
Specify these properties on the command line or via the build.properties file. For example: Specify these properties on the command line or via the build.properties file. For example:
+sqoop.test.mysql.connectstring.host_url=jdbc:mysql://host.example.com/+ sqoop.test.mysql.connectstring.host_url=jdbc:mysql://host.example.com/
+sqoop.test.mysql.databasename=MYDB+ sqoop.test.mysql.databasename=MYDB
+sqoop.test.mysql.username=MYUSR+ sqoop.test.mysql.username=MYUSR
+sqoop.test.mysql.password=MYPWD+ sqoop.test.mysql.password=MYPWD
If not specified, the default value used for this property is: If not specified, the default value used for this property is:
+jdbc:mysql://127.0.0.1:13306/+ jdbc:mysql://localhost/
===== Oracle ===== Oracle
Install Oracle Enterprise Edition 10.2.0+. Instructions for configuring the Install Oracle Enterprise Edition 10.2.0+. Instructions for configuring the
database are in +OracleManagerTest+. Download the ojdbc6_g jar. database are in OracleManagerTest. Download the ojdbc6_g jar.
If running the tests against Oracle XE (Express Edition) - a lot of them will If running the tests against Oracle XE (Express Edition) - a lot of them will
fail as it does not include the partitioning feature. fail as it does not include the partitioning feature.
Use the following system properties to configure connection to the Oracle XE host used for testing: Use the following system properties to configure connection to the Oracle host used for testing:
+sqoop.test.oracle.connectstring+, +sqoop.test.oracle.username+ and +sqoop.test.oracle.password+. sqoop.test.oracle.connectstring, sqoop.test.oracle.username and sqoop.test.oracle.password.
You can configure the connection properties separately for tests requiring Oracle EE:
+sqoop.test.oracle-ee.connectstring+, +sqoop.test.oracle-ee.username+ and +sqoop.test.oracle-ee.password+.
Specify these properties on the command line or via the build.properties file. For example: Specify these properties on the command line or via the build.properties file. For example:
+sqoop.test.oracle.connectstring=jdbc:oracle:thin:@//host.example.com/xe+ sqoop.test.oracle.connectstring=jdbc:oracle:thin:@//host.example.com/xe
+sqoop.test.oracle.username=MYUSR+ sqoop.test.oracle.username=MYUSR
+sqoop.test.oracle.password=MYPWD+ sqoop.test.oracle.password=MYPWD
If not specified, the default value used for +sqoop.test.oracle.connectstring+ property is: If not specified, the default value used for this property is:
+jdbc:oracle:thin:@//localhost:1521/xe+ jdbc:oracle:thin:@//localhost/xe
The default value used for +sqoop.test.oracle-ee.connectstring+ property is:
+jdbc:oracle:thin:@//localhost:1522/sqoop+
Users sqooptest and sqooptest2 should be created prior to running the tests. Users sqooptest and sqooptest2 should be created prior to running the tests.
SQL script is available in src/test/oraoop/create_users.sql SQL script is available in src/test/oraoop/create_users.sql
@ -222,104 +209,106 @@ SQL script is available in src/test/oraoop/create_users.sql
===== PostgreSQL ===== PostgreSQL
Install PostgreSQL 8.3.9. Download the postgresql 8.4 jdbc driver. Instructions Install PostgreSQL 8.3.9. Download the postgresql 8.4 jdbc driver. Instructions
for configuring the database are in +PostgresqlTest+. for configuring the database are in PostgresqlTest.
Use the following system properties to configure connection to the PostgreSQL host used for testing: Use the following system properties to configure connection to the PostgreSQL host used for testing:
+sqoop.test.postgresql.connectstring.host_url+, +sqoop.test.postgresql.database+, +sqoop.test.postgresql.username+ and sqoop.test.postgresql.connectstring.host_url, sqoop.test.postgresql.database, sqoop.test.postgresql.username and
+sqoop.test.postgresql.password+. sqoop.test.postgresql.password.
Specify this property on the Specify this property on the
command line or via the build.properties file. For example: command line or via the build.properties file. For example:
+sqoop.test.postgresql.connectstring.host_url=jdbc:postgresql://sqoop-dbs.sf.cloudera.com/+ sqoop.test.postgresql.connectstring.host_url=jdbc:postgresql://sqoop-dbs.sf.cloudera.com/
+sqoop.test.postgresql.database=MYDB+ sqoop.test.postgresql.database=MYDB
+sqoop.test.postgresql.username=MYUSR+ sqoop.test.postgresql.username=MYUSR
+sqoop.test.postgresql.password=MYPWD+ sqoop.test.postgresql.password=MYPWD
If not specified, the default value used for this property is: If not specified, the default value used for this property is:
+jdbc:postgresql://localhost:15432/+ jdbc:postgresql://localhost/
===== SQL Server ===== SQL Server
Install SQL Server Express 2012 and create a database instance and Install SQL Server Express 2012 and create a database instance and
download the appropriate JDBC driver. Instructions for configuring the download the appropriate JDBC driver. Instructions for configuring the
database can be found in +SQLServerManagerImportManualTest+. database can be found in SQLServerManagerImportManualTest.
Use the following system properties to configure connection to the SQL Server host used for testing: Use the following system properties to configure connection to the SQL Server host used for testing:
+sqoop.test.sqlserver.connectstring.host_url+, +sqoop.test.sqlserver.database+, +ms.sqlserver.username+ and sqoop.test.sqlserver.connectstring.host_url, sqoop.test.sqlserver.database, sqoop.test.sqlserver.username and
+ms.sqlserver.password+. sqoop.test.sqlserver.password.
the URL for the SQL Server host used for testing. Specify this property on the the URL for the SQL Server host used for testing. Specify this property on the
command line or via the build.properties file. For example: command line or via the build.properties file. For example:
+sqoop.test.sqlserver.connectstring.host_url=jdbc:sqlserver://sqlserverhost:1433+ sqoop.test.sqlserver.connectstring.host_url=jdbc:sqlserver://sqlserverhost:1433
+sqoop.test.sqlserver.database=MYDB+ sqoop.test.sqlserver.database=MYDB
+ms.sqlserver.username=MYUSR+ ms.sqlserver.username=MYUSR
+ms.sqlserver.password=MYPWD+ ms.sqlserver.password=MYPWD
If not specified, the default value used for this property is: If not specified, the default value used for this property is:
+jdbc:sqlserver://localhost:1433+ jdbc:sqlserver://sqlserverhost:1433
This can be useful if you have the hostname sqlserverhost mapped to the IP
address of the SQL Server instance.
===== Cubrid ===== Cubrid
Install Cubrid 9.2.2.0003 and create a database instance and download the Install Cubrid 9.2.2.0003 and create a database instance and download the
appropriate JDBC driver. Instructions for configuring the database are in appropriate JDBC driver. Instructions for configuring the database are in
+CubridAuthTest+, +CubridCompatTest+, +CubridManagerImportTest+ CubridAuthTest, CubridCompatTest, CubridManagerImportTest
and +CubridManagerExportTest+. and CubridManagerExportTest.
Use the following system properties to configure connection to the Cubrid host used for testing: Use the following system properties to configure connection to the Cubrid host used for testing:
+sqoop.test.cubrid.connectstring.host_url+, +sqoop.test.cubrid.connectstring.database+, sqoop.test.cubrid.connectstring.host_url, sqoop.test.cubrid.connectstring.database,
+sqoop.test.cubrid.connectstring.username+ and +sqoop.test.cubrid.connectstring.password+. sqoop.test.cubrid.connectstring.username and sqoop.test.cubrid.connectstring.password.
Specify this property on the command Specify this property on the command
line or via the build.properties file. For example: line or via the build.properties file. For example:
+sqoop.test.cubrid.connectstring.host_url=jdbc:cubrid:localhost+ sqoop.test.cubrid.connectstring.host_url=jdbc:cubrid:localhost
+sqoop.test.cubrid.connectstring.database=MYDB+ sqoop.test.cubrid.connectstring.database=MYDB
+sqoop.test.cubrid.connectstring.username=MYUSR+ sqoop.test.cubrid.connectstring.username=MYUSR
+sqoop.test.cubrid.connectstring.password=MYPWD+ sqoop.test.cubrid.connectstring.password=MYPWD
If not specified, the default value used for this property is: If not specified, the default value used for this property is:
+jdbc:cubrid:localhost:33000+ jdbc:cubrid:localhost
===== DB2 ===== DB2
Install DB2 9.74 Express C and download the appropriate JDBC driver. Install DB2 9.74 Express C and download the appropriate JDBC driver.
Instructions for configuring the server can be found in Instructions for configuring the server can be found in
+DB2ManagerImportManualTest+. DB2ManagerImportManualTest.
Use the following system properties to configure connection to the DB2 host used for testing: Use the following system properties to configure connection to the DB2 host used for testing:
+sqoop.test.db2.connectstring.host_url+, +sqoop.test.db2.connectstring.database+, +sqoop.test.db2.connectstring.username+ and sqoop.test.db2.connectstring.host_url, sqoop.test.db2.connectstring.database, sqoop.test.db2.connectstring.username and
+sqoop.test.db2.connectstring.password+. sqoop.test.db2.connectstring.password.
Specify this property on Specify this property on
the command line or via build.properties file. For example: the command line or via build.properties file. For example:
+sqoop.test.db2.connectstring.host_url=jdbc:db2://db2host:50000+ sqoop.test.db2.connectstring.host_url=jdbc:db2://db2host:50000
+sqoop.test.db2.connectstring.database=MYDB+ sqoop.test.db2.connectstring.database=MYDB
+sqoop.test.db2.connectstring.username=MYUSR+ sqoop.test.db2.connectstring.username=MYUSR
+sqoop.test.db2.connectstring.password=MYPWD+ sqoop.test.db2.connectstring.password=MYPWD
If not specified, the default value used for this property is: If not specified, the default value used for this property is:
+jdbc:db2://localhost:50000+ jdbc:db2://db2host:50000
This can be useful if you have the hostname db2host mapped to the IP
address of the DB2 Server instance.
==== Running the Third-party Tests on native database servers ==== Running the Third-party Tests on native database servers
After the third-party databases are installed and configured, run: After the third-party databases are installed and configured, run:
---- ++++
ant test -Dthirdparty=true -Dsqoop.thirdparty.lib.dir=/path/to/jdbc/drivers/ [-DconnectionProperty1=propertyValue1 ant test -Dthirdparty=true -Dsqoop.thirdparty.lib.dir=/path/to/jdbc/drivers/
-DconnectionProperty2=propertyValue ...] ./gradlew -Dsqoop.thirdparty.lib.dir=/relative/path/to/jdbc/drivers/ thirdPartyTest
./gradlew -Dsqoop.thirdparty.lib.dir=/relative/path/to/jdbc/drivers/ thirdPartyTest [-DconnectionProperty1=propertyValue1 ++++
-DconnectionProperty2=propertyValue ...]
----
This command will run all thirdparty tests except some DB2 tests. This command will run all thirdparty tests except some DB2 tests.
To run these DB2 test, specify the property "manual" instead of "thirdparty" To run these DB2 test, specify the property "manual" instead of "thirdparty"
as follows: as follows:
---- ++++
ant test -Dmanual=true -Dsqoop.thirdparty.lib.dir=/path/to/jdbc/drivers/ [-DconnectionProperty1=propertyValue1 ant test -Dmanual=true -Dsqoop.thirdparty.lib.dir=/path/to/jdbc/drivers/
-DconnectionProperty2=propertyValue ...] ./gradlew -Dsqoop.thirdparty.lib.dir=/relative/path/to/jdbc/drivers/ manualTest
./gradlew -Dsqoop.thirdparty.lib.dir=/relative/path/to/jdbc/drivers/ manualTest [-DconnectionProperty1=propertyValue1 ++++
-DconnectionProperty2=propertyValue ...]
----
Note that +sqoop.thirdparty.lib.dir+ can also be specified in Note that +sqoop.thirdparty.lib.dir+ can also be specified in
+build.properties+. +build.properties+.
@ -337,11 +326,11 @@ https://docs.docker.com/compose/install/
===== Downloading docker images ===== Downloading docker images
MySQL, PostgreSQL, MSSQL, DB2, Oracle XE and Cubrid images are freely available on Docker Hub so they will be pulled automatically by the startup command specified below however the Oracle EE image has to be built manually. Please refer to the README.md file on the below Github project for building instructions: MySQL, PostgreSQL, MSSQL, DB2 and Cubrid images are freely available on Docker Hub so they will be pulled automatically by the startup command specified below however the Oracle EE image has to be built manually. Please refer to the README.md file on the below Github project for building instructions:
https://github.com/oracle/docker-images/tree/master/OracleDatabase https://github.com/oracle/docker-images/tree/master/OracleDatabase
Please note that some Sqoop third party tests require Oracle Enterprise Edition and the startup command assumes version 12.2.0.1. Please note that Sqoop third party tests require Oracle Enterprise Edition and the startup command assumes version 12.2.0.1.
===== Starting the Docker containers ===== Starting the Docker containers
@ -361,8 +350,7 @@ If it is executed without parameters it starts the following services:
|+postgresql+ |PostgreSQL 9.6.4 |+postgresql+ |PostgreSQL 9.6.4
|+mssql+ |MSSQL 14.0.1000.169 Developer Edition |+mssql+ |MSSQL 14.0.1000.169 Developer Edition
|+cubrid+ |Cubrid 10.0 |+cubrid+ |Cubrid 10.0
|+oracle-ee+ |Oracle EE 12.2.0.1 |+oracle+ |Oracle EE 12.2.0.1
|+oracle+ |Oracle XE 11g Release 2
|+db2+ |DB2 Express Edition 10.5.0.5-3.10.0 |+db2+ |DB2 Express Edition 10.5.0.5-3.10.0
|====================== |======================
@ -379,8 +367,7 @@ After the startup script is executed the containers need some time to initialize
158e5421d134 ibmcom/db2express-c:10.5.0.5-3.10.0 "/home/db2inst1/db..." 43 seconds ago Up 40 seconds (health: starting) 22/tcp, 0.0.0.0:50000->50000/tcp sqoop_db2_container 158e5421d134 ibmcom/db2express-c:10.5.0.5-3.10.0 "/home/db2inst1/db..." 43 seconds ago Up 40 seconds (health: starting) 22/tcp, 0.0.0.0:50000->50000/tcp sqoop_db2_container
---- ----
Most of the containers need less than 1 minute to start up but DB2 needs \~5 minutes and Oracle EE needs \~15 minutes. Most of the containers need less than 1 minute to start up but DB2 needs ~5 minutes and Oracle needs ~15 minutes. The Docker images need ~17GB free disk space and Docker requires ~5GB of memory to start all of them at the same time.
The Docker images need \~17GB free disk space and Docker requires ~5GB of memory to start all of them at the same time.
===== Stopping the Docker containers ===== Stopping the Docker containers
@ -397,6 +384,31 @@ You can execute the third party tests against the DBs running in Docker containe
---- ----
ant clean test -Dthirdparty=true -Dsqoop.thirdparty.lib.dir=<path_to_thirdparty_lib_directory> \ ant clean test -Dthirdparty=true -Dsqoop.thirdparty.lib.dir=<path_to_thirdparty_lib_directory> \
-Dsqoop.test.mysql.connectstring.host_url=jdbc:mysql://127.0.0.1:3306/ \
-Dsqoop.test.mysql.databasename=sqoop \
-Dsqoop.test.mysql.password=Sqoop12345 \
-Dsqoop.test.mysql.username=sqoop \
-Dsqoop.test.oracle.connectstring=jdbc:oracle:thin:@//localhost:1521/sqoop \
-Dsqoop.test.oracle.username=SYSTEM \
-Dsqoop.test.oracle.password=Sqoop12345 \
-Dsqoop.test.postgresql.connectstring.host_url=jdbc:postgresql://localhost/ \
-Dsqoop.test.postgresql.database=sqoop \
-Dsqoop.test.postgresql.username=sqoop \
-Dsqoop.test.postgresql.password=Sqoop12345 \
-Dsqoop.test.cubrid.connectstring.host_url=jdbc:cubrid:localhost:33000 \
-Dsqoop.test.cubrid.connectstring.username=sqoop \
-Dsqoop.test.cubrid.connectstring.database=sqoop \
-Dsqoop.test.cubrid.connectstring.password=Sqoop12345 \
-Dmapred.child.java.opts="\-Djava.security.egd=file:/dev/../dev/urandom" \
-Dtest.timeout=10000000 \
-Dsqoop.test.sqlserver.connectstring.host_url=jdbc:sqlserver://localhost:1433 \
-Dsqoop.test.sqlserver.database=master \
-Dms.sqlserver.username=sa \
-Dms.sqlserver.password=Sqoop12345 \
-Dsqoop.test.db2.connectstring.host_url=jdbc:db2://localhost:50000 \
-Dsqoop.test.db2.connectstring.database=SQOOP \
-Dsqoop.test.db2.connectstring.username=DB2INST1 \
-Dsqoop.test.db2.connectstring.password=Sqoop12345 \
-Ds3.bucket.url=<your-bucket-url> \ -Ds3.bucket.url=<your-bucket-url> \
-Ds3.generator.command=<your-credential-generator-command> -Ds3.generator.command=<your-credential-generator-command>
---- ----
@ -405,14 +417,36 @@ or
---- ----
./gradlew -Dsqoop.thirdparty.lib.dir=<path_to_thirdparty_lib_directory> \ ./gradlew -Dsqoop.thirdparty.lib.dir=<path_to_thirdparty_lib_directory> \
-Dsqoop.test.mysql.connectstring.host_url=jdbc:mysql://127.0.0.1:3306/ \
-Dsqoop.test.mysql.databasename=sqoop \
-Dsqoop.test.mysql.password=Sqoop12345 \
-Dsqoop.test.mysql.username=sqoop \
-Dsqoop.test.oracle.connectstring=jdbc:oracle:thin:@//localhost:1521/sqoop \
-Dsqoop.test.oracle.username=SYSTEM \
-Dsqoop.test.oracle.password=Sqoop12345 \
-Dsqoop.test.postgresql.connectstring.host_url=jdbc:postgresql://localhost/ \
-Dsqoop.test.postgresql.database=sqoop \
-Dsqoop.test.postgresql.username=sqoop \
-Dsqoop.test.postgresql.password=Sqoop12345 \
-Dsqoop.test.cubrid.connectstring.host_url=jdbc:cubrid:localhost:33000 \
-Dsqoop.test.cubrid.connectstring.username=sqoop \
-Dsqoop.test.cubrid.connectstring.database=sqoop \
-Dsqoop.test.cubrid.connectstring.password=Sqoop12345 \
-Dmapred.child.java.opts="\-Djava.security.egd=file:/dev/../dev/urandom" \
-Dtest.timeout=10000000 \
-Dsqoop.test.sqlserver.connectstring.host_url=jdbc:sqlserver://localhost:1433 \
-Dsqoop.test.sqlserver.database=master \
-Dms.sqlserver.username=sa \
-Dms.sqlserver.password=Sqoop12345 \
-Dsqoop.test.db2.connectstring.host_url=jdbc:db2://localhost:50000 \
-Dsqoop.test.db2.connectstring.database=SQOOP \
-Dsqoop.test.db2.connectstring.username=DB2INST1 \
-Dsqoop.test.db2.connectstring.password=Sqoop12345 \
-Ds3.bucket.url=<your-bucket-url> \ -Ds3.bucket.url=<your-bucket-url> \
-Ds3.generator.command=<your-credential-generator-command> \ -Ds3.generator.command=<your-credential-generator-command> \
thirdPartyTest thirdPartyTest
---- ----
Gradle is able download all of the necessary drivers except the Oracle one but the Ant build has to be provided
with the drivers for all the databases.
Please note that even if you do not need to install RDBMSs to run Sqoop third party tests against the Docker containers you still need to install the following tools: Please note that even if you do not need to install RDBMSs to run Sqoop third party tests against the Docker containers you still need to install the following tools:
* mysqldump * mysqldump
@ -618,19 +652,3 @@ export XML_CATALOG_FILES=/usr/local/etc/xml/catalog
* +./gradle build -x test+ for skipping a single test or a set of tests * +./gradle build -x test+ for skipping a single test or a set of tests
* +./gradlew dependencyInsight --configuration optionalConfiguration --dependency searchedForDependency+ to get a dependency tree * +./gradlew dependencyInsight --configuration optionalConfiguration --dependency searchedForDependency+ to get a dependency tree
* +./gradlew dependencies+ to get the list of the dependencies of the selected project, broken down by configuration * +./gradlew dependencies+ to get the list of the dependencies of the selected project, broken down by configuration
== Setting up Travis CI
You can now set up a Travis CI job for your own Sqoop fork so you can easily test your patches before uploading them for review.
The steps for setting up the CI job are the following:
- Go to https://github.com/apache/sqoop and fork the project if it is not already forked for your GitHub account.
- Go to https://travis-ci.com and Sign up with GitHub.
- Accept the Authorization of Travis CI. You’ll be redirected to GitHub.
- Click the green Activate button, and select the Sqoop repository.
The Travis CI job uses Gradle to build and test the project and it is able to execute all of the tests except the
ones requiring an Oracle EE database.
Note that by default the Oracle XE tests are also disabled because the necessary +ojdbc6.jar+ is not publicly available.
Contributors have to upload +ojdbc6.jar+ to a public drive and make it available to the CI job by setting
the +ORACLE_DRIVER_URL+ environment variable in Travis (for more information see Travis
https://docs.travis-ci.com/user/environment-variables/#defining-variables-in-repository-settings[documentation]).

100
build.gradle
View File

@ -38,14 +38,6 @@ repositories {
maven { maven {
url 'http://conjars.org/repo/' url 'http://conjars.org/repo/'
} }
maven {
url 'http://maven.cubrid.org/'
}
// DB2 driver repository
maven {
url 'https://artifacts.alfresco.com/nexus/content/repositories/public/'
}
} }
sourceSets { sourceSets {
@ -160,12 +152,6 @@ dependencies {
testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoallVersion testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoallVersion
testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: systemRulesVersion testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: systemRulesVersion
testCompile group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion, ext: 'jar' testCompile group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion, ext: 'jar'
// JDBC drivers
testCompile group: 'mysql', name: 'mysql-connector-java', version: mySqlConnectorVersion
testCompile group: 'cubrid', name: 'cubrid-jdbc', version: cubridJdbcDriverVersion
testCompile group: 'com.microsoft.sqlserver', name: 'mssql-jdbc', version: msSqlJdbcDriverVersion
testCompile group: 'com.ibm.db2.jcc', name: 'db2jcc4', version: db2JdbcDriverVersion
} }
task unitTest (type: Test) { task unitTest (type: Test) {
@ -200,92 +186,6 @@ task thirdPartyTest (type: Test) {
systemProperty "ms.datatype.test.hdfsprefix" ,"file:///" systemProperty "ms.datatype.test.hdfsprefix" ,"file:///"
} }
task postgreSqlTest (type: Test) {
description 'Run Postgresql third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task mySqlTest (type: Test) {
description 'Run MySql third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.MysqlTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task cubridTest (type: Test) {
description 'Run Cubrid third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.CubridTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task db2Test (type: Test) {
description 'Run DB2 third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.Db2Test'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task mainFrameTest (type: Test) {
description 'Run Mainframe third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.MainFrameTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task oracleTest (type: Test) {
description 'Run Oracle third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task oracleXeTest (type: Test) {
description 'Run Oracle third party tests which can be run with Oracle Express Edition too.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleTest'
excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task oracleEeTest (type: Test) {
description 'Run Oracle third party tests which require Oracle Enterprise Edition.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task sqlServerTest (type: Test) {
description 'Run MS SqlServer third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.SqlServerTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
systemProperty "ms.datatype.test.data.file.export" ,"DatatypeTestData-export-lite.txt"
systemProperty "ms.datatype.test.data.file.import" ,"DatatypeTestData-import-lite.txt"
systemProperty "ms.datatype.test.data.file.delim" ,","
systemProperty "ms.datatype.test.hdfsprefix" ,"file:///"
}
test { test {
description 'Run tests that do not need external JDBC driver and/or a docker container. ' + description 'Run tests that do not need external JDBC driver and/or a docker container. ' +
'This is the same as running unitTest, integrationTest and kerberizedTest.' 'This is the same as running unitTest, integrationTest and kerberizedTest.'

61
build.xml
View File

@ -185,63 +185,59 @@
value="${name}/[conf]/[artifact]-[revision](-[classifier]).[ext]"/> value="${name}/[conf]/[artifact]-[revision](-[classifier]).[ext]"/>
<!--test related properties --> <!--test related properties -->
<property name="sqoop.test.oracle.connectstring" value="jdbc:oracle:thin:@//localhost:1521/xe"/> <property name="sqoop.test.oracle.connectstring" value="jdbc:oracle:thin:@//localhost/xe"/>
<property name="sqoop.test.oracle.username" value="SYSTEM"/> <property name="sqoop.test.oracle.username" value="SQOOPTEST"/>
<property name="sqoop.test.oracle.password" value="oracle"/> <property name="sqoop.test.oracle.password" value="12345"/>
<property name="sqoop.test.oracle-ee.connectstring" value="jdbc:oracle:thin:@//localhost:1522/sqoop"/>
<property name="sqoop.test.oracle-ee.username" value="SYSTEM"/>
<property name="sqoop.test.oracle-ee.password" value="Sqoop12345"/>
<property name="sqoop.test.mysql.connectstring.host_url" <property name="sqoop.test.mysql.connectstring.host_url"
value="jdbc:mysql://127.0.0.1:13306/"/> value="jdbc:mysql://localhost/"/>
<property name="sqoop.test.mysql.username" <property name="sqoop.test.mysql.username"
value="sqoop"/> value="${env.USER}"/>
<property name="sqoop.test.mysql.password" value="Sqoop12345" /> <property name="sqoop.test.mysql.password" value="" />
<property name="sqoop.test.mysql.databasename" <property name="sqoop.test.mysql.databasename"
value="sqoop"/> value="sqooptestdb"/>
<property name="sqoop.test.cubrid.connectstring.host_url" <property name="sqoop.test.cubrid.connectstring.host_url"
value="jdbc:cubrid:localhost:33000"/> value="jdbc:cubrid:localhost:30000"/>
<property name="sqoop.test.cubrid.connectstring.database" value="sqoop" /> <property name="sqoop.test.cubrid.connectstring.database" value="SQOOPCUBRIDTEST" />
<property name="sqoop.test.cubrid.connectstring.username" value="sqoop" /> <property name="sqoop.test.cubrid.connectstring.username" value="SQOOPUSER" />
<property name="sqoop.test.cubrid.connectstring.password" value="Sqoop12345" /> <property name="sqoop.test.cubrid.connectstring.password" value="PASSWORD" />
<property name="sqoop.test.postgresql.connectstring.host_url" <property name="sqoop.test.postgresql.connectstring.host_url"
value="jdbc:postgresql://localhost:15432/"/> value="jdbc:postgresql://localhost/"/>
<property name="sqoop.test.postgresql.database" <property name="sqoop.test.postgresql.database"
value="sqoop" /> value="sqooptest" />
<property name="sqoop.test.postgresql.tablespace" <property name="sqoop.test.postgresql.tablespace"
value="sqoop" /> value="sqooptest" />
<property name="sqoop.test.postgresql.username" <property name="sqoop.test.postgresql.username"
value="sqoop" /> value="sqooptest" />
<property name="sqoop.test.postgresql.password" <property name="sqoop.test.postgresql.password"
value="Sqoop12345" /> value="" />
<property name="sqoop.test.postgresql.pg_bulkload" <property name="sqoop.test.postgresql.pg_bulkload"
value="pg_bulkload" /> value="pg_bulkload" />
<property name="sqoop.test.sqlserver.connectstring.host_url" <property name="sqoop.test.sqlserver.connectstring.host_url"
value="jdbc:sqlserver://localhost:1433"/> value="jdbc:sqlserver://sqlserverhost:1433"/>
<property name="sqoop.test.sqlserver.database" <property name="sqoop.test.sqlserver.database"
value="master"/> value="sqooptest"/>
<property name="ms.sqlserver.username" <property name="ms.sqlserver.username"
value="sa"/> value="SQOOPUSER"/>
<property name="ms.sqlserver.password" <property name="ms.sqlserver.password"
value="Sqoop12345"/> value="PASSWORD"/>
<property name="sqoop.test.db2.connectstring.host_url" value="jdbc:db2://localhost:50000" /> <property name="sqoop.test.db2.connectstring.host_url" value="jdbc:db2://db2host:50000" />
<property name="sqoop.test.db2.connectstring.database" value="SQOOP" /> <property name="sqoop.test.db2.connectstring.database" value="SQOOP" />
<property name="sqoop.test.db2.connectstring.username" value="DB2INST1" /> <property name="sqoop.test.db2.connectstring.username" value="SQOOP" />
<property name="sqoop.test.db2.connectstring.password" value="Sqoop12345" /> <property name="sqoop.test.db2.connectstring.password" value="SQOOP" />
<property name="sqoop.test.netezza.host" value="nz-host" /> <property name="sqoop.test.netezza.host" value="nz-host" />
<property name="sqoop.test.netezza.port" value="5480" /> <property name="sqoop.test.netezza.port" value="5480" />
@ -839,15 +835,6 @@
<sysproperty key="sqoop.test.oracle.password" <sysproperty key="sqoop.test.oracle.password"
value="${sqoop.test.oracle.password}"/> value="${sqoop.test.oracle.password}"/>
<sysproperty key="sqoop.test.oracle-ee.connectstring"
value="${sqoop.test.oracle-ee.connectstring}"/>
<sysproperty key="sqoop.test.oracle-ee.username"
value="${sqoop.test.oracle-ee.username}"/>
<sysproperty key="sqoop.test.oracle-ee.password"
value="${sqoop.test.oracle-ee.password}"/>
<sysproperty key="sqoop.test.mysql.connectstring.host_url" <sysproperty key="sqoop.test.mysql.connectstring.host_url"
value="${sqoop.test.mysql.connectstring.host_url}"/> value="${sqoop.test.mysql.connectstring.host_url}"/>
@ -962,7 +949,7 @@
excludes="**/${test.exclude}.java,**/KerberizedTest.java,**/IntegrationTest.java,**/SqoopTest.java, excludes="**/${test.exclude}.java,**/KerberizedTest.java,**/IntegrationTest.java,**/SqoopTest.java,
**/UnitTest.java,**/CubridTest.java,**/Db2Test.java,**/MainFrameTest.java,**/MysqlTest.java, **/UnitTest.java,**/CubridTest.java,**/Db2Test.java,**/MainFrameTest.java,**/MysqlTest.java,
**/NetezzaTest.java,**/OracleTest.java,**/PostgresqlTest.java,**/S3Test.java,**/SqlServerTest.java, **/NetezzaTest.java,**/OracleTest.java,**/PostgresqlTest.java,**/S3Test.java,**/SqlServerTest.java,
**/ThirdPartyTest.java,**/OracleEeTest.java"/> **/ThirdPartyTest.java"/>
</batchtest> </batchtest>
<batchtest todir="${build.test}" if="testcase"> <batchtest todir="${build.test}" if="testcase">
<fileset dir="${test.dir}" includes="**/${testcase}.java"/> <fileset dir="${test.dir}" includes="**/${testcase}.java"/>

5
gradle.properties
View File

@ -51,11 +51,6 @@ mockitoallVersion=1.9.5
systemRulesVersion=1.17.0 systemRulesVersion=1.17.0
assertjVersion=2.8.0 assertjVersion=2.8.0
mySqlConnectorVersion=5.1.47
cubridJdbcDriverVersion=10.1.2.7694
msSqlJdbcDriverVersion=7.0.0.jre8
db2JdbcDriverVersion=10.1
checkstyleVersion=5.5 checkstyleVersion=5.5
version=1.5.0-SNAPSHOT version=1.5.0-SNAPSHOT

19
src/scripts/thirdpartytest/docker-compose/oraclescripts/ee-healthcheck.sh
View File

@ -1,19 +0,0 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
echo "SELECT 1 from DUAL;" | sqlplus -L SQOOPTEST2/ABCDEF@//localhost:1521/sqoop

2
src/scripts/thirdpartytest/docker-compose/oraclescripts/healthcheck.sh
View File

@ -16,4 +16,4 @@
# limitations under the License. # limitations under the License.
echo "SELECT 1 from DUAL;" | sqlplus -L SQOOPTEST2/ABCDEF@//localhost:1521/xe echo "SELECT 1 from DUAL;" | sqlplus -L SQOOPTEST2/ABCDEF@//localhost:1521/sqoop

3
src/scripts/thirdpartytest/docker-compose/oraclescripts/startup/.cache
View File

@ -1,3 +0,0 @@
d41d8cd98f00b204e9800998ecf8427e -
a941f0f5fbfcb329f646d544d40de691 /docker-entrypoint-initdb.d/oracleusersetup.sql
d41d8cd98f00b204e9800998ecf8427e -

27
src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml
View File

@ -25,7 +25,7 @@ services:
MYSQL_PASSWORD: Sqoop12345 MYSQL_PASSWORD: Sqoop12345
MYSQL_DATABASE: sqoop MYSQL_DATABASE: sqoop
ports: ports:
- 13306:3306 - 3306:3306
command: command:
mysqld --sql_mode="ANSI_QUOTES" mysqld --sql_mode="ANSI_QUOTES"
healthcheck: healthcheck:
@ -42,7 +42,7 @@ services:
POSTGRES_USER: sqoop POSTGRES_USER: sqoop
POSTGRES_PASSWORD: Sqoop12345 POSTGRES_PASSWORD: Sqoop12345
ports: ports:
- 15432:5432 - 5432:5432
healthcheck: healthcheck:
test: ["CMD", "pg_isready"] test: ["CMD", "pg_isready"]
interval: 30s interval: 30s
@ -77,37 +77,22 @@ services:
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 10 retries: 10
oracle-ee: oracle:
image: oracle/database:12.2.0.1-ee image: oracle/database:12.2.0.1-ee
container_name: sqoop_oracle-ee_container container_name: sqoop_oracle_container
environment: environment:
COMPOSE_HTTP_TIMEOUT: 200 COMPOSE_HTTP_TIMEOUT: 200
ORACLE_SID: sqoop ORACLE_SID: sqoop
ORACLE_PWD: Sqoop12345 ORACLE_PWD: Sqoop12345
ports: ports:
- 1522:1521 - 1521:1521
volumes: volumes:
- ./oraclescripts:/opt/oracle/scripts - ./oraclescripts:/opt/oracle/scripts
healthcheck: healthcheck:
test: ["CMD", "/opt/oracle/scripts/ee-healthcheck.sh"] test: ["CMD", "/opt/oracle/scripts/healthcheck.sh"]
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 60 retries: 60
oracle:
image: sath89/oracle-xe-11g
container_name: sqoop_oracle_container
environment:
COMPOSE_HTTP_TIMEOUT: 200
ports:
- 1521:1521
volumes:
- ./oraclescripts:/opt/oracle/scripts
- ./oraclescripts/startup:/docker-entrypoint-initdb.d
healthcheck:
test: ["CMD", "/opt/oracle/scripts/healthcheck.sh"]
interval: 5s
timeout: 5s
retries: 50
db2: db2:
image: ibmcom/db2express-c:10.5.0.5-3.10.0 image: ibmcom/db2express-c:10.5.0.5-3.10.0
container_name: sqoop_db2_container container_name: sqoop_db2_container

8
src/test/org/apache/sqoop/manager/cubrid/CubridTestUtils.java
View File

@ -51,17 +51,17 @@ public class CubridTestUtils {
public static final String HOST_URL = System public static final String HOST_URL = System
.getProperty("sqoop.test.cubrid.connectstring.host_url", .getProperty("sqoop.test.cubrid.connectstring.host_url",
"jdbc:cubrid:localhost:33000"); "jdbc:cubrid:localhost:30000");
static final String TEST_DATABASE = System static final String TEST_DATABASE = System
.getProperty("sqoop.test.cubrid.connectstring.database", .getProperty("sqoop.test.cubrid.connectstring.database",
"sqoop"); "SQOOPCUBRIDTEST");
static final String TEST_USER = System static final String TEST_USER = System
.getProperty("sqoop.test.cubrid.connectstring.username", .getProperty("sqoop.test.cubrid.connectstring.username",
"sqoop"); "SQOOPUSER");
static final String TEST_PASS = System static final String TEST_PASS = System
.getProperty("sqoop.test.cubrid.connectstring.password", .getProperty("sqoop.test.cubrid.connectstring.password",
"Sqoop12345"); "PASSWORD");
static final String TABLE_NAME = "EMPLOYEES_CUBRID"; static final String TABLE_NAME = "EMPLOYEES_CUBRID";
static final String NULL_TABLE_NAME = "NULL_EMPLOYEES_CUBRID"; static final String NULL_TABLE_NAME = "NULL_EMPLOYEES_CUBRID";
static final String CONNECT_STRING = HOST_URL + ":" static final String CONNECT_STRING = HOST_URL + ":"

20
src/test/org/apache/sqoop/manager/db2/DB2ImportAllTableWithSchemaManualTest.java
View File

@ -51,10 +51,6 @@
import org.apache.sqoop.util.LoggingUtils; import org.apache.sqoop.util.LoggingUtils;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
import static org.apache.sqoop.manager.db2.DB2TestUtils.CONNECT_STRING;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_PASSWORD;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_USER;
import static org.apache.sqoop.manager.db2.DB2TestUtils.HOST_URL;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
@ -85,8 +81,24 @@ public class DB2ImportAllTableWithSchemaManualTest extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog( public static final Log LOG = LogFactory.getLog(
DB2ImportAllTableWithSchemaManualTest.class.getName()); DB2ImportAllTableWithSchemaManualTest.class.getName());
static final String HOST_URL = System.getProperty(
"sqoop.test.db2.connectstring.host_url",
"jdbc:db2://9.30.245.234:60000");
static final String DATABASE_NAME = System.getProperty(
"sqoop.test.db2.connectstring.database",
"TESTDB");
static final String DATABASE_USER = System.getProperty(
"sqoop.test.db2.connectstring.username",
"DB2FENC1");
static final String DATABASE_PASSWORD = System.getProperty(
"sqoop.test.db2.connectstring.password",
"DB2FENC1");
static final String TABLE_NAME = "TEST.COMPANY"; static final String TABLE_NAME = "TEST.COMPANY";
static final String TABLE_SCHEMA = "TEST"; static final String TABLE_SCHEMA = "TEST";
static final String CONNECT_STRING = HOST_URL
+ "/" + DATABASE_NAME;
static String ExpectedResults = static String ExpectedResults =
"1,doc1"; "1,doc1";

19
src/test/org/apache/sqoop/manager/db2/DB2ManagerImportManualTest.java
View File

@ -44,9 +44,6 @@
import org.apache.sqoop.util.FileListing; import org.apache.sqoop.util.FileListing;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
import static org.apache.sqoop.manager.db2.DB2TestUtils.CONNECT_STRING;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_PASSWORD;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_USER;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
@ -77,8 +74,24 @@ public class DB2ManagerImportManualTest extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog( public static final Log LOG = LogFactory.getLog(
DB2ManagerImportManualTest.class.getName()); DB2ManagerImportManualTest.class.getName());
static final String HOST_URL = System.getProperty(
"sqoop.test.db2.connectstring.host_url",
"jdbc:db2://db2host:50000");
static final String DATABASE_NAME = System.getProperty(
"sqoop.test.db2.connectstring.database",
"SQOOP");
static final String DATABASE_USER = System.getProperty(
"sqoop.test.db2.connectstring.username",
"SQOOP");
static final String DATABASE_PASSWORD = System.getProperty(
"sqoop.test.db2.connectstring.password",
"SQOOP");
static final String TABLE_NAME = "EMPLOYEES_DB2"; static final String TABLE_NAME = "EMPLOYEES_DB2";
static final String QUALIFIED_TABLE_NAME = DATABASE_USER + ".EMPLOYEES_DB2"; static final String QUALIFIED_TABLE_NAME = DATABASE_USER + ".EMPLOYEES_DB2";
static final String CONNECT_STRING = HOST_URL
+ "/" + DATABASE_NAME
+ ":currentSchema=" + DATABASE_USER +";";
static { static {
LOG.info("Using DB2 CONNECT_STRING: " + CONNECT_STRING); LOG.info("Using DB2 CONNECT_STRING: " + CONNECT_STRING);

28
src/test/org/apache/sqoop/manager/db2/DB2TestUtils.java
View File

@ -1,28 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.manager.db2;
public class DB2TestUtils {
public static final String HOST_URL = System.getProperty("sqoop.test.db2.connectstring.host_url", "jdbc:db2://localhost:50000");
public static final String DATABASE_NAME = System.getProperty("sqoop.test.db2.connectstring.database", "SQOOP");
public static final String DATABASE_USER = System.getProperty("sqoop.test.db2.connectstring.username", "DB2INST1");
public static final String DATABASE_PASSWORD = System.getProperty("sqoop.test.db2.connectstring.password", "Sqoop12345");
public static final String CONNECT_STRING = HOST_URL + "/" + DATABASE_NAME + ":currentSchema=" + DATABASE_USER +";";
}

20
src/test/org/apache/sqoop/manager/db2/DB2XmlTypeImportManualTest.java
View File

@ -44,10 +44,6 @@
import org.apache.sqoop.util.FileListing; import org.apache.sqoop.util.FileListing;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
import static org.apache.sqoop.manager.db2.DB2TestUtils.CONNECT_STRING;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_PASSWORD;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_USER;
import static org.apache.sqoop.manager.db2.DB2TestUtils.HOST_URL;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
@ -78,7 +74,23 @@ public class DB2XmlTypeImportManualTest extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog( public static final Log LOG = LogFactory.getLog(
DB2XmlTypeImportManualTest.class.getName()); DB2XmlTypeImportManualTest.class.getName());
static final String HOST_URL = System.getProperty(
"sqoop.test.db2.connectstring.host_url",
"jdbc:db2://db2host:60000");
static final String DATABASE_NAME = System.getProperty(
"sqoop.test.db2.connectstring.database",
"SQOOP");
static final String DATABASE_USER = System.getProperty(
"sqoop.test.db2.connectstring.username",
"SQOOP");
static final String DATABASE_PASSWORD = System.getProperty(
"sqoop.test.db2.connectstring.password",
"SQOOP");
static final String TABLE_NAME = "COMPANY"; static final String TABLE_NAME = "COMPANY";
static final String CONNECT_STRING = HOST_URL
+ "/" + DATABASE_NAME;
static final String HIVE_TABLE_NAME = "HCOMPANY"; static final String HIVE_TABLE_NAME = "HCOMPANY";
static String ExpectedResults = static String ExpectedResults =
"1,doc1,<company name=\"Company1\"><emp id=\"31201\" salary=\"60000\" gender=\"Female\"><name><first>Laura </first><last>Brown</last></name><dept id=\"M25\">Finance</dept></emp></company>"; "1,doc1,<company name=\"Company1\"><emp id=\"31201\" salary=\"60000\" gender=\"Female\"><name><first>Laura </first><last>Brown</last></name><dept id=\"M25\">Finance</dept></emp></company>";

16
src/test/org/apache/sqoop/manager/mysql/MySQLTestUtils.java
View File

@ -43,18 +43,18 @@ public final class MySQLTestUtils {
private String userName; private String userName;
private String userPass; private String userPass;
private String mysqlDbName; private String mysqlDbNAme;
private String mySqlConnectString; private String mySqlConnectString;
public MySQLTestUtils() { public MySQLTestUtils() {
hostUrl = System.getProperty( hostUrl = System.getProperty(
"sqoop.test.mysql.connectstring.host_url", "sqoop.test.mysql.connectstring.host_url",
"jdbc:mysql://127.0.0.1:13306/"); "jdbc:mysql://localhost/");
userName = System.getProperty("sqoop.test.mysql.username", "sqoop"); userName = System.getProperty("sqoop.test.mysql.username", getCurrentUser());
userPass = System.getProperty("sqoop.test.mysql.password", "Sqoop12345"); userPass = System.getProperty("sqoop.test.mysql.password");
mysqlDbName = System.getProperty("sqoop.test.mysql.databasename", "sqoop"); mysqlDbNAme = System.getProperty("sqoop.test.mysql.databasename", "sqooptestdb");
mySqlConnectString = getHostUrl() + getMysqlDbName(); mySqlConnectString = getHostUrl() + getMysqlDbNAme();
} }
public String getHostUrl() { public String getHostUrl() {
@ -69,8 +69,8 @@ public String getUserPass() {
return userPass; return userPass;
} }
public String getMysqlDbName() { public String getMysqlDbNAme() {
return mysqlDbName; return mysqlDbNAme;
} }

15
src/test/org/apache/sqoop/manager/oracle/ExportTest.java
View File

@ -18,19 +18,15 @@
package org.apache.sqoop.manager.oracle; package org.apache.sqoop.manager.oracle;
import org.apache.sqoop.manager.oracle.util.OracleUtils;
import org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
/** /**
* Test exporting data into Oracle. * Test exporting data into Oracle.
*/ */
@Category(OracleEeTest.class)
public class ExportTest extends OraOopTestCase { public class ExportTest extends OraOopTestCase {
private static final ExportTest TEST_CASE = new ExportTest(); private static final ExportTest TEST_CASE = new ExportTest();
@ -71,15 +67,4 @@ public static void cleanUpHdfsData() throws Exception {
TEST_CASE.closeTestEnvConnection(); TEST_CASE.closeTestEnvConnection();
} }
protected String getConnectString() {
return org.apache.sqoop.manager.oracle.util.OracleUtils.EE_CONNECT_STRING;
}
protected String getUsername() {
return org.apache.sqoop.manager.oracle.util.OracleUtils.ORACLE_EE_USER_NAME;
}
protected String getPassword() {
return OracleUtils.ORACLE_EE_USER_PASS;
}
} }

15
src/test/org/apache/sqoop/manager/oracle/ImportTest.java
View File

@ -21,17 +21,13 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.manager.oracle.OraOopConstants. import org.apache.sqoop.manager.oracle.OraOopConstants.
OraOopOracleDataChunkMethod; OraOopOracleDataChunkMethod;
import org.apache.sqoop.manager.oracle.util.OracleUtils;
import org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
/** /**
* Test import data from Oracle. * Test import data from Oracle.
*/ */
@Category(OracleEeTest.class)
public class ImportTest extends OraOopTestCase { public class ImportTest extends OraOopTestCase {
private static final boolean DISABLE_ORACLE_ESCAPING_FLAG = false; private static final boolean DISABLE_ORACLE_ESCAPING_FLAG = false;
@ -279,15 +275,4 @@ public void testProductPartIotImport() throws Exception {
} }
} }
protected String getConnectString() {
return OracleUtils.EE_CONNECT_STRING;
}
protected String getUsername() {
return OracleUtils.ORACLE_EE_USER_NAME;
}
protected String getPassword() {
return OracleUtils.ORACLE_EE_USER_PASS;
}
} }

27
src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java
View File

@ -108,7 +108,9 @@ protected void setSqoopGenClassName(String sqoopGenClassName) {
protected Connection getTestEnvConnection() throws SQLException { protected Connection getTestEnvConnection() throws SQLException {
if (this.conn == null) { if (this.conn == null) {
this.conn = DriverManager.getConnection(getConnectString(), getUsername(), getPassword()); this.conn =
DriverManager.getConnection(OracleUtils.CONNECT_STRING,
OracleUtils.ORACLE_USER_NAME, OracleUtils.ORACLE_USER_PASS);
this.conn.setAutoCommit(false); this.conn.setAutoCommit(false);
} }
return this.conn; return this.conn;
@ -229,13 +231,13 @@ protected int runImport(String tableName, Configuration sqoopConf,
} }
sqoopArgs.add("--connect"); sqoopArgs.add("--connect");
sqoopArgs.add(getConnectString()); sqoopArgs.add(OracleUtils.CONNECT_STRING);
sqoopArgs.add("--username"); sqoopArgs.add("--username");
sqoopArgs.add(getUsername()); sqoopArgs.add(OracleUtils.ORACLE_USER_NAME);
sqoopArgs.add("--password"); sqoopArgs.add("--password");
sqoopArgs.add(getPassword()); sqoopArgs.add(OracleUtils.ORACLE_USER_PASS);
sqoopArgs.add("--table"); sqoopArgs.add("--table");
sqoopArgs.add(tableName); sqoopArgs.add(tableName);
@ -307,13 +309,13 @@ protected int runExport(String tableName, List<String> additionalArgs, Map<Strin
sqoopArgs.add("--direct"); sqoopArgs.add("--direct");
sqoopArgs.add("--connect"); sqoopArgs.add("--connect");
sqoopArgs.add(getConnectString()); sqoopArgs.add(OracleUtils.CONNECT_STRING);
sqoopArgs.add("--username"); sqoopArgs.add("--username");
sqoopArgs.add(getUsername()); sqoopArgs.add(OracleUtils.ORACLE_USER_NAME);
sqoopArgs.add("--password"); sqoopArgs.add("--password");
sqoopArgs.add(getPassword()); sqoopArgs.add(OracleUtils.ORACLE_USER_PASS);
sqoopArgs.add("--table"); sqoopArgs.add("--table");
sqoopArgs.add(tableName); sqoopArgs.add(tableName);
@ -374,15 +376,4 @@ protected void cleanupFolders() throws Exception {
HadoopFiles.delete(new Path(getSqoopGenLibDirectory()), true); HadoopFiles.delete(new Path(getSqoopGenLibDirectory()), true);
} }
protected String getConnectString() {
return OracleUtils.CONNECT_STRING;
}
protected String getUsername() {
return OracleUtils.ORACLE_USER_NAME;
}
protected String getPassword() {
return OracleUtils.ORACLE_USER_PASS;
}
} }

16
src/test/org/apache/sqoop/manager/oracle/OraOopTypesTest.java
View File

@ -24,12 +24,8 @@
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.Statement; import java.sql.Statement;
import org.apache.sqoop.manager.oracle.util.OracleUtils;
import org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category(OracleEeTest.class)
public class OraOopTypesTest extends OraOopTestCase { public class OraOopTypesTest extends OraOopTestCase {
@Test @Test
public void ensureTypesAfterExportMappedAsExpected() throws Exception { public void ensureTypesAfterExportMappedAsExpected() throws Exception {
@ -161,16 +157,4 @@ public void ensureTypesAfterExportMappedAsExpected() throws Exception {
closeTestEnvConnection(); closeTestEnvConnection();
} }
} }
protected String getConnectString() {
return OracleUtils.EE_CONNECT_STRING;
}
protected String getUsername() {
return OracleUtils.ORACLE_EE_USER_NAME;
}
protected String getPassword() {
return OracleUtils.ORACLE_EE_USER_PASS;
}
} }

4
src/test/org/apache/sqoop/manager/oracle/OracleConnectionFactoryTest.java
View File

@ -105,8 +105,8 @@ private void setAndCheckJdbcFetchSize(int jdbcFetchSize) {
+ "statement being used to check the JDBC fetch size."); + "statement being used to check the JDBC fetch size.");
} }
if (rowsPerFetch < jdbcFetchSize * 0.925 if (rowsPerFetch < jdbcFetchSize * 0.95
|| rowsPerFetch > jdbcFetchSize * 1.075) { || rowsPerFetch > jdbcFetchSize * 1.05) {
Assert Assert
.fail(String .fail(String
.format( .format(

15
src/test/org/apache/sqoop/manager/oracle/TimestampDataTest.java
View File

@ -19,10 +19,7 @@
package org.apache.sqoop.manager.oracle; package org.apache.sqoop.manager.oracle;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.manager.oracle.util.OracleUtils;
import org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
@ -31,7 +28,6 @@
* requires the tests to be run in a different process. Maven needs to be setup * requires the tests to be run in a different process. Maven needs to be setup
* to fork per test class. * to fork per test class.
*/ */
@Category(OracleEeTest.class)
public class TimestampDataTest extends OraOopTestCase { public class TimestampDataTest extends OraOopTestCase {
private static final boolean DISABLE_ORACLE_ESCAPING_FLAG = false; private static final boolean DISABLE_ORACLE_ESCAPING_FLAG = false;
@ -54,15 +50,4 @@ public void testProductImportTimezone() throws Exception {
} }
} }
protected String getConnectString() {
return org.apache.sqoop.manager.oracle.util.OracleUtils.EE_CONNECT_STRING;
}
protected String getUsername() {
return org.apache.sqoop.manager.oracle.util.OracleUtils.ORACLE_EE_USER_NAME;
}
protected String getPassword() {
return OracleUtils.ORACLE_EE_USER_PASS;
}
} }

11
src/test/org/apache/sqoop/manager/oracle/util/OracleUtils.java
View File

@ -36,13 +36,12 @@ public final class OracleUtils {
public static final Log LOG = LogFactory.getLog(OracleUtils.class.getName()); public static final Log LOG = LogFactory.getLog(OracleUtils.class.getName());
public static final String CONNECT_STRING = System.getProperty("sqoop.test.oracle.connectstring", "jdbc:oracle:thin:@//localhost:1521/xe"); // Express edition hardcoded name.
public static final String ORACLE_USER_NAME = System.getProperty("sqoop.test.oracle.username", "SYSTEM"); public static final String ORACLE_DATABASE_NAME = "xe";
public static final String ORACLE_USER_PASS = System.getProperty("sqoop.test.oracle.password", "oracle");
public static final String EE_CONNECT_STRING = System.getProperty("sqoop.test.oracle-ee.connectstring", "jdbc:oracle:thin:@//localhost:1522/sqoop"); public static final String CONNECT_STRING = System.getProperty("sqoop.test.oracle.connectstring", "jdbc:oracle:thin:@//localhost/" + ORACLE_DATABASE_NAME);
public static final String ORACLE_EE_USER_NAME = System.getProperty("sqoop.test.oracle-ee.username", "SYSTEM"); public static final String ORACLE_USER_NAME = System.getProperty("sqoop.test.oracle.username", "SQOOPTEST");
public static final String ORACLE_EE_USER_PASS = System.getProperty("sqoop.test.oracle-ee.password", "Sqoop12345"); public static final String ORACLE_USER_PASS = System.getProperty("sqoop.test.oracle.password", "12345");
public static final String ORACLE_SECONDARY_USER_NAME = "SQOOPTEST2"; public static final String ORACLE_SECONDARY_USER_NAME = "SQOOPTEST2";
public static final String ORACLE_SECONDARY_USER_PASS = "ABCDEF"; public static final String ORACLE_SECONDARY_USER_PASS = "ABCDEF";

17
src/test/org/apache/sqoop/manager/postgresql/DirectPostgreSQLExportManualTest.java
View File

@ -18,10 +18,6 @@
package org.apache.sqoop.manager.postgresql; package org.apache.sqoop.manager.postgresql;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.CONNECT_STRING;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.DATABASE_USER;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.PASSWORD;
import java.io.IOException; import java.io.IOException;
import java.sql.Connection; import java.sql.Connection;
import java.sql.SQLException; import java.sql.SQLException;
@ -67,6 +63,17 @@ public class DirectPostgreSQLExportManualTest extends TestExport {
LogFactory.getLog(DirectPostgreSQLExportManualTest.class.getName()); LogFactory.getLog(DirectPostgreSQLExportManualTest.class.getName());
private DBConfiguration dbConf; private DBConfiguration dbConf;
static final String HOST_URL =
System.getProperty("sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
static final String DATABASE =
System.getProperty("sqoop.test.postgresql.database", "sqooptest");
static final String USERNAME =
System.getProperty("sqoop.test.postgresql.username", "sqooptest");
static final String PASSWORD = System.getProperty(
"sqoop.test.postgresql.password");
static final String CONNECT_STRING = HOST_URL + DATABASE;
public DirectPostgreSQLExportManualTest() { public DirectPostgreSQLExportManualTest() {
JobConf conf = new JobConf(getConf()); JobConf conf = new JobConf(getConf());
DBConfiguration.configureDB(conf, DBConfiguration.configureDB(conf,
@ -88,7 +95,7 @@ protected String getConnectString() {
} }
protected String getUserName() { protected String getUserName() {
return DATABASE_USER; return USERNAME;
} }
@Override @Override

15
src/test/org/apache/sqoop/manager/postgresql/PGBulkloadManagerManualTest.java
View File

@ -18,9 +18,6 @@
package org.apache.sqoop.manager.postgresql; package org.apache.sqoop.manager.postgresql;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.CONNECT_STRING;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.DATABASE_USER;
import java.io.IOException; import java.io.IOException;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
@ -74,10 +71,18 @@ public class PGBulkloadManagerManualTest extends TestExport {
public static final Log LOG = public static final Log LOG =
LogFactory.getLog(PGBulkloadManagerManualTest.class.getName()); LogFactory.getLog(PGBulkloadManagerManualTest.class.getName());
private DBConfiguration dbConf; private DBConfiguration dbConf;
static final String HOST_URL =
System.getProperty("sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
static final String DATABASE =
System.getProperty("sqoop.test.postgresql.database", "sqooptest");
static final String TABLESPACE = static final String TABLESPACE =
System.getProperty("sqoop.test.postgresql.tablespace", "sqoop"); System.getProperty("sqoop.test.postgresql.tablespace", "sqooptest");
static final String USERNAME =
System.getProperty("sqoop.test.postgresql.username", "sqooptest");
static final String PG_BULKLOAD = static final String PG_BULKLOAD =
System.getProperty("sqoop.test.postgresql.pg_bulkload", "pg_bulkload"); System.getProperty("sqoop.test.postgresql.pg_bulkload", "pg_bulkload");
static final String CONNECT_STRING = HOST_URL + DATABASE;
public PGBulkloadManagerManualTest() { public PGBulkloadManagerManualTest() {
JobConf conf = new JobConf(getConf()); JobConf conf = new JobConf(getConf());
@ -103,7 +108,7 @@ protected String getConnectString() {
protected String getUserName() { protected String getUserName() {
return DATABASE_USER; return USERNAME;
} }

17
src/test/org/apache/sqoop/manager/postgresql/PostgresqlExportTest.java
View File

@ -17,10 +17,6 @@
*/ */
package org.apache.sqoop.manager.postgresql; package org.apache.sqoop.manager.postgresql;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.CONNECT_STRING;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.DATABASE_USER;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.PASSWORD;
import org.apache.sqoop.SqoopOptions; import org.apache.sqoop.SqoopOptions;
import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest; import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest;
import org.apache.sqoop.testutil.CommonArgs; import org.apache.sqoop.testutil.CommonArgs;
@ -56,11 +52,24 @@ public class PostgresqlExportTest extends ExportJobTestCase {
public static final Log LOG = LogFactory.getLog( public static final Log LOG = LogFactory.getLog(
PostgresqlExportTest.class.getName()); PostgresqlExportTest.class.getName());
static final String HOST_URL = System.getProperty(
"sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
static final String DATABASE_USER = System.getProperty(
"sqoop.test.postgresql.username",
"sqooptest");
static final String DATABASE_NAME = System.getProperty(
"sqoop.test.postgresql.database",
"sqooptest");
static final String PASSWORD = System.getProperty(
"sqoop.test.postgresql.password");
static final String TABLE_NAME = "EMPLOYEES_PG"; static final String TABLE_NAME = "EMPLOYEES_PG";
static final String PROCEDURE_NAME = "INSERT_AN_EMPLOYEE"; static final String PROCEDURE_NAME = "INSERT_AN_EMPLOYEE";
static final String STAGING_TABLE_NAME = "STAGING"; static final String STAGING_TABLE_NAME = "STAGING";
static final String SCHEMA_PUBLIC = "public"; static final String SCHEMA_PUBLIC = "public";
static final String SCHEMA_SPECIAL = "special"; static final String SCHEMA_SPECIAL = "special";
static final String CONNECT_STRING = HOST_URL + DATABASE_NAME;
protected Connection connection; protected Connection connection;

11
src/test/org/apache/sqoop/manager/postgresql/PostgresqlExternalTableImportTest.java
View File

@ -17,9 +17,6 @@
*/ */
package org.apache.sqoop.manager.postgresql; package org.apache.sqoop.manager.postgresql;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.CONNECT_STRING;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.DATABASE_USER;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.PASSWORD;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
@ -57,6 +54,13 @@ public class PostgresqlExternalTableImportTest extends ImportJobTestCase {
public static final Log LOG = LogFactory public static final Log LOG = LogFactory
.getLog(PostgresqlExternalTableImportTest.class.getName()); .getLog(PostgresqlExternalTableImportTest.class.getName());
static final String HOST_URL = System.getProperty("sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
static final String DATABASE_USER = System.getProperty(
"sqoop.test.postgresql.username", "sqooptest");
static final String DATABASE_NAME = System.getProperty(
"sqoop.test.postgresql.database", "sqooptest");
static final String PASSWORD = System.getProperty("sqoop.test.postgresql.password");
static final String TABLE_NAME = "EMPLOYEES_PG"; static final String TABLE_NAME = "EMPLOYEES_PG";
static final String NULL_TABLE_NAME = "NULL_EMPLOYEES_PG"; static final String NULL_TABLE_NAME = "NULL_EMPLOYEES_PG";
@ -64,6 +68,7 @@ public class PostgresqlExternalTableImportTest extends ImportJobTestCase {
static final String DIFFERENT_TABLE_NAME = "DIFFERENT_TABLE"; static final String DIFFERENT_TABLE_NAME = "DIFFERENT_TABLE";
static final String SCHEMA_PUBLIC = "public"; static final String SCHEMA_PUBLIC = "public";
static final String SCHEMA_SPECIAL = "special"; static final String SCHEMA_SPECIAL = "special";
static final String CONNECT_STRING = HOST_URL + DATABASE_NAME;
static final String EXTERNAL_TABLE_DIR = "/tmp/external/employees_pg"; static final String EXTERNAL_TABLE_DIR = "/tmp/external/employees_pg";
protected Connection connection; protected Connection connection;

8
src/test/org/apache/sqoop/manager/postgresql/PostgresqlTestUtil.java
View File

@ -22,20 +22,20 @@ public class PostgresqlTestUtil {
public static final String HOST_URL = System.getProperty( public static final String HOST_URL = System.getProperty(
"sqoop.test.postgresql.connectstring.host_url", "sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost:15432/"); "jdbc:postgresql://localhost/");
public static final String DATABASE_USER = System.getProperty( public static final String DATABASE_USER = System.getProperty(
"sqoop.test.postgresql.username", "sqoop.test.postgresql.username",
"sqoop"); "sqooptest");
public static final String DATABASE_NAME = System.getProperty( public static final String DATABASE_NAME = System.getProperty(
"sqoop.test.postgresql.database", "sqoop.test.postgresql.database",
"sqoop"); "sqooptest");
public static final String CONNECT_STRING = HOST_URL + DATABASE_NAME; public static final String CONNECT_STRING = HOST_URL + DATABASE_NAME;
public static final String PASSWORD = System.getProperty( public static final String PASSWORD = System.getProperty(
"sqoop.test.postgresql.password", "Sqoop12345"); "sqoop.test.postgresql.password");
static final String TABLE_NAME = "EMPLOYEES_PG"; static final String TABLE_NAME = "EMPLOYEES_PG";

8
src/test/org/apache/sqoop/manager/sqlserver/MSSQLTestUtils.java
View File

@ -38,15 +38,15 @@ public class MSSQLTestUtils {
MSSQLTestUtils.class.getName()); MSSQLTestUtils.class.getName());
public static final String DATABASE_USER = System.getProperty( public static final String DATABASE_USER = System.getProperty(
"ms.sqlserver.username", "sa"); "ms.sqlserver.username", "SQOOPUSER");
public static final String DATABASE_PASSWORD = System.getProperty( public static final String DATABASE_PASSWORD = System.getProperty(
"ms.sqlserver.password", "Sqoop12345"); "ms.sqlserver.password", "PASSWORD");
public static final String DATABASE_NAME = System.getProperty( public static final String DATABASE_NAME = System.getProperty(
"sqoop.test.sqlserver.database", "sqoop.test.sqlserver.database",
"master"); "sqooptest");
public static final String HOST_URL = System.getProperty( public static final String HOST_URL = System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url", "sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://localhost:1433"); "jdbc:sqlserver://sqlserverhost:1433");
public static final String CONNECT_STRING = HOST_URL + ";database=" + DATABASE_NAME; public static final String CONNECT_STRING = HOST_URL + ";database=" + DATABASE_NAME;

17
src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerExportTest.java
View File

@ -42,10 +42,6 @@
import java.sql.Statement; import java.sql.Statement;
import java.util.ArrayList; import java.util.ArrayList;
import static org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.DATABASE_NAME;
import static org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.DATABASE_USER;
import static org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.DATABASE_PASSWORD;
import static org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.HOST_URL;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
@ -58,6 +54,19 @@ public class SQLServerManagerExportTest extends ExportJobTestCase {
public static final Log LOG = LogFactory.getLog( public static final Log LOG = LogFactory.getLog(
SQLServerManagerExportTest.class.getName()); SQLServerManagerExportTest.class.getName());
static final String HOST_URL = System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
static final String DATABASE_NAME = System.getProperty(
"sqoop.test.sqlserver.database",
"sqooptest");
static final String DATABASE_USER = System.getProperty(
"ms.sqlserver.username",
"sqoopuser");
static final String DATABASE_PASSWORD = System.getProperty(
"ms.sqlserver.password",
"password");
static final String SCHEMA_DBO = "dbo"; static final String SCHEMA_DBO = "dbo";
static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL"; static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL";
static final String DBO_BINARY_TABLE_NAME = "BINARYTYPE_MSSQL"; static final String DBO_BINARY_TABLE_NAME = "BINARYTYPE_MSSQL";

17
src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java
View File

@ -52,10 +52,6 @@
import java.sql.Statement; import java.sql.Statement;
import java.util.Arrays; import java.util.Arrays;
import static org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.DATABASE_NAME;
import static org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.DATABASE_USER;
import static org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.DATABASE_PASSWORD;
import static org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.HOST_URL;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
@ -90,6 +86,19 @@ public class SQLServerManagerImportTest extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog( public static final Log LOG = LogFactory.getLog(
SQLServerManagerImportTest.class.getName()); SQLServerManagerImportTest.class.getName());
static final String HOST_URL = System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
static final String DATABASE_NAME = System.getProperty(
"sqoop.test.sqlserver.database",
"sqooptest");
static final String DATABASE_USER = System.getProperty(
"ms.sqlserver.username",
"sqoopuser");
static final String DATABASE_PASSWORD = System.getProperty(
"ms.sqlserver.password",
"password");
static final String SCHEMA_DBO = "dbo"; static final String SCHEMA_DBO = "dbo";
static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL"; static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL";
static final String SCHEMA_SCH = "sch"; static final String SCHEMA_SCH = "sch";

21
src/test/org/apache/sqoop/metastore/db2/DB2JobToolTest.java
View File

@ -18,10 +18,6 @@
package org.apache.sqoop.metastore.db2; package org.apache.sqoop.metastore.db2;
import static org.apache.sqoop.manager.db2.DB2TestUtils.CONNECT_STRING;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_PASSWORD;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_USER;
import org.apache.sqoop.metastore.JobToolTestBase; import org.apache.sqoop.metastore.JobToolTestBase;
import org.apache.sqoop.testcategories.thirdpartytest.Db2Test; import org.apache.sqoop.testcategories.thirdpartytest.Db2Test;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
@ -48,6 +44,23 @@
@Category(Db2Test.class) @Category(Db2Test.class)
public class DB2JobToolTest extends JobToolTestBase { public class DB2JobToolTest extends JobToolTestBase {
private static final String HOST_URL = System.getProperty(
"sqoop.test.db2.connectstring.host_url",
"jdbc:db2://db2host:50000");
private static final String DATABASE_NAME = System.getProperty(
"sqoop.test.db2.connectstring.database",
"SQOOP");
private static final String DATABASE_USER = System.getProperty(
"sqoop.test.db2.connectstring.username",
"SQOOP");
private static final String DATABASE_PASSWORD = System.getProperty(
"sqoop.test.db2.connectstring.password",
"SQOOP");
private static final String CONNECT_STRING = HOST_URL
+ "/" + DATABASE_NAME
+ ":currentSchema=" + DATABASE_USER +";";
public DB2JobToolTest() { public DB2JobToolTest() {
super(CONNECT_STRING, DATABASE_USER, DATABASE_PASSWORD); super(CONNECT_STRING, DATABASE_USER, DATABASE_PASSWORD);
} }

21
src/test/org/apache/sqoop/metastore/db2/DB2MetaConnectIncrementalImportTest.java
View File

@ -18,10 +18,6 @@
package org.apache.sqoop.metastore.db2; package org.apache.sqoop.metastore.db2;
import static org.apache.sqoop.manager.db2.DB2TestUtils.CONNECT_STRING;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_PASSWORD;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_USER;
import org.apache.sqoop.metastore.MetaConnectIncrementalImportTestBase; import org.apache.sqoop.metastore.MetaConnectIncrementalImportTestBase;
import org.apache.sqoop.testcategories.thirdpartytest.Db2Test; import org.apache.sqoop.testcategories.thirdpartytest.Db2Test;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
@ -48,6 +44,23 @@
@Category(Db2Test.class) @Category(Db2Test.class)
public class DB2MetaConnectIncrementalImportTest extends MetaConnectIncrementalImportTestBase { public class DB2MetaConnectIncrementalImportTest extends MetaConnectIncrementalImportTestBase {
private static final String HOST_URL = System.getProperty(
"sqoop.test.db2.connectstring.host_url",
"jdbc:db2://db2host:50000");
private static final String DATABASE_NAME = System.getProperty(
"sqoop.test.db2.connectstring.database",
"SQOOP");
private static final String DATABASE_USER = System.getProperty(
"sqoop.test.db2.connectstring.username",
"SQOOP");
private static final String DATABASE_PASSWORD = System.getProperty(
"sqoop.test.db2.connectstring.password",
"SQOOP");
private static final String CONNECT_STRING = HOST_URL
+ "/" + DATABASE_NAME
+ ":currentSchema=" + DATABASE_USER +";";
public DB2MetaConnectIncrementalImportTest() { public DB2MetaConnectIncrementalImportTest() {
super(CONNECT_STRING, DATABASE_USER, DATABASE_PASSWORD); super(CONNECT_STRING, DATABASE_USER, DATABASE_PASSWORD);
} }

21
src/test/org/apache/sqoop/metastore/db2/DB2SavedJobsTest.java
View File

@ -18,10 +18,6 @@
package org.apache.sqoop.metastore.db2; package org.apache.sqoop.metastore.db2;
import static org.apache.sqoop.manager.db2.DB2TestUtils.CONNECT_STRING;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_PASSWORD;
import static org.apache.sqoop.manager.db2.DB2TestUtils.DATABASE_USER;
import org.apache.sqoop.metastore.SavedJobsTestBase; import org.apache.sqoop.metastore.SavedJobsTestBase;
import org.apache.sqoop.manager.JdbcDrivers; import org.apache.sqoop.manager.JdbcDrivers;
import org.apache.sqoop.testcategories.thirdpartytest.Db2Test; import org.apache.sqoop.testcategories.thirdpartytest.Db2Test;
@ -49,6 +45,23 @@
@Category(Db2Test.class) @Category(Db2Test.class)
public class DB2SavedJobsTest extends SavedJobsTestBase { public class DB2SavedJobsTest extends SavedJobsTestBase {
private static final String HOST_URL = System.getProperty(
"sqoop.test.db2.connectstring.host_url",
"jdbc:db2://db2host:50000");
private static final String DATABASE_NAME = System.getProperty(
"sqoop.test.db2.connectstring.database",
"SQOOP");
private static final String DATABASE_USER = System.getProperty(
"sqoop.test.db2.connectstring.username",
"SQOOP");
private static final String DATABASE_PASSWORD = System.getProperty(
"sqoop.test.db2.connectstring.password",
"SQOOP");
private static final String CONNECT_STRING = HOST_URL
+ "/" + DATABASE_NAME
+ ":currentSchema=" + DATABASE_USER +";";
public DB2SavedJobsTest() { public DB2SavedJobsTest() {
super(CONNECT_STRING, DATABASE_USER, DATABASE_PASSWORD, JdbcDrivers.DB2.getDriverClass()); super(CONNECT_STRING, DATABASE_USER, DATABASE_PASSWORD, JdbcDrivers.DB2.getDriverClass());
} }

13
src/test/org/apache/sqoop/metastore/postgres/PostgresJobToolTest.java
View File

@ -18,10 +18,6 @@
package org.apache.sqoop.metastore.postgres; package org.apache.sqoop.metastore.postgres;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.CONNECT_STRING;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.DATABASE_USER;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.PASSWORD;
import org.apache.sqoop.metastore.JobToolTestBase; import org.apache.sqoop.metastore.JobToolTestBase;
import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest; import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
@ -44,6 +40,15 @@
@Category(PostgresqlTest.class) @Category(PostgresqlTest.class)
public class PostgresJobToolTest extends JobToolTestBase { public class PostgresJobToolTest extends JobToolTestBase {
private static final String HOST_URL = System.getProperty("sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
private static final String DATABASE_USER = System.getProperty(
"sqoop.test.postgresql.username", "sqooptest");
private static final String DATABASE_NAME = System.getProperty(
"sqoop.test.postgresql.database", "sqooptest");
private static final String PASSWORD = System.getProperty("sqoop.test.postgresql.password");
private static final String CONNECT_STRING = HOST_URL + DATABASE_NAME;
public PostgresJobToolTest() { public PostgresJobToolTest() {
super(CONNECT_STRING, DATABASE_USER, PASSWORD); super(CONNECT_STRING, DATABASE_USER, PASSWORD);
} }

13
src/test/org/apache/sqoop/metastore/postgres/PostgresMetaConnectIncrementalImportTest.java
View File

@ -18,10 +18,6 @@
package org.apache.sqoop.metastore.postgres; package org.apache.sqoop.metastore.postgres;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.CONNECT_STRING;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.DATABASE_USER;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.PASSWORD;
import org.apache.sqoop.metastore.MetaConnectIncrementalImportTestBase; import org.apache.sqoop.metastore.MetaConnectIncrementalImportTestBase;
import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest; import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
@ -44,6 +40,15 @@
@Category(PostgresqlTest.class) @Category(PostgresqlTest.class)
public class PostgresMetaConnectIncrementalImportTest extends MetaConnectIncrementalImportTestBase { public class PostgresMetaConnectIncrementalImportTest extends MetaConnectIncrementalImportTestBase {
private static final String HOST_URL = System.getProperty("sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
private static final String DATABASE_USER = System.getProperty(
"sqoop.test.postgresql.username", "sqooptest");
private static final String DATABASE_NAME = System.getProperty(
"sqoop.test.postgresql.database", "sqooptest");
private static final String PASSWORD = System.getProperty("sqoop.test.postgresql.password");
private static final String CONNECT_STRING = HOST_URL + DATABASE_NAME;
public PostgresMetaConnectIncrementalImportTest() { public PostgresMetaConnectIncrementalImportTest() {
super(CONNECT_STRING, DATABASE_USER, PASSWORD); super(CONNECT_STRING, DATABASE_USER, PASSWORD);
} }

13
src/test/org/apache/sqoop/metastore/postgres/PostgresSavedJobsTest.java
View File

@ -18,10 +18,6 @@
package org.apache.sqoop.metastore.postgres; package org.apache.sqoop.metastore.postgres;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.CONNECT_STRING;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.DATABASE_USER;
import static org.apache.sqoop.manager.postgresql.PostgresqlTestUtil.PASSWORD;
import org.apache.sqoop.metastore.SavedJobsTestBase; import org.apache.sqoop.metastore.SavedJobsTestBase;
import org.apache.sqoop.manager.JdbcDrivers; import org.apache.sqoop.manager.JdbcDrivers;
import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest; import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest;
@ -45,6 +41,15 @@
@Category(PostgresqlTest.class) @Category(PostgresqlTest.class)
public class PostgresSavedJobsTest extends SavedJobsTestBase { public class PostgresSavedJobsTest extends SavedJobsTestBase {
private static final String HOST_URL = System.getProperty("sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
private static final String DATABASE_USER = System.getProperty(
"sqoop.test.postgresql.username", "sqooptest");
private static final String DATABASE_NAME = System.getProperty(
"sqoop.test.postgresql.database", "sqooptest");
private static final String PASSWORD = System.getProperty("sqoop.test.postgresql.password");
private static final String CONNECT_STRING = HOST_URL + DATABASE_NAME;
public PostgresSavedJobsTest() { public PostgresSavedJobsTest() {
super(CONNECT_STRING, DATABASE_USER, PASSWORD, JdbcDrivers.POSTGRES.getDriverClass()); super(CONNECT_STRING, DATABASE_USER, PASSWORD, JdbcDrivers.POSTGRES.getDriverClass());
} }

26
src/test/org/apache/sqoop/testcategories/thirdpartytest/OracleEeTest.java
View File

@ -1,26 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.testcategories.thirdpartytest;
/**
* An Oracle EE test shall test scenarios where a Oracle Enterprise Edition database is needed,
* for example because of partitioning.
*/
public interface OracleEeTest extends OracleTest {
}