github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 20:40:58 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3055: Fixing MySQL tests failing due to

Browse Source 
ignored test inputs/configuration

(Anna Szonyi via Attila Szabo)
This commit is contained in:
Attila Szabo 2016-11-29 15:33:16 +01:00
parent b0b7b4779c
commit 48cecdc99e
11 changed files with 151 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
build.xml
View File

@ -372,6 +372,12 @@
<property name="sqoop.test.mysql.connectstring.host_url"
value="jdbc:mysql://localhost/"/>
<property name="sqoop.test.mysql.username"
value="${env.USER}"/>
<property name="sqoop.test.mysql.password" value="" />
<property name="sqoop.test.mysql.databasename"
value="sqooptestdb"/>
<property name="sqoop.test.cubrid.connectstring.host_url"
value="jdbc:cubrid:localhost:30000"/>
<property name="sqoop.test.cubrid.connectstring.database" value="SQOOPCUBRIDTEST" />
@ -983,6 +989,15 @@
<sysproperty key="sqoop.test.mysql.connectstring.host_url"
value="${sqoop.test.mysql.connectstring.host_url}"/>
<sysproperty key="sqoop.test.mysql.username"
value="${sqoop.test.mysql.username}"/>
<sysproperty key="sqoop.test.mysql.password"
value="${sqoop.test.mysql.password}"/>
<sysproperty key="sqoop.test.mysql.databasename"
value="${sqoop.test.mysql.databasename}"/>
<sysproperty key="sqoop.test.cubrid.connectstring.host_url"
value="${sqoop.test.cubrid.connectstring.host_url}"/>

27
src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
View File

@ -39,9 +39,6 @@
import com.cloudera.sqoop.TestExport;
import com.cloudera.sqoop.mapreduce.MySQLExportMapper;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Test the DirectMySQLManager implementation's exportJob() functionality.
*/
@ -55,6 +52,7 @@ public class DirectMySQLExportTest extends TestExport {
// instance variables populated during setUp, used during tests.
private DirectMySQLManager manager;
private Connection conn;
private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
@Override
protected Connection getConnection() {
@ -68,7 +66,7 @@ protected boolean useHsqldbTestServer() {
@Override
protected String getConnectString() {
return MySQLTestUtils.CONNECT_STRING;
return mySQLTestUtils.getMySqlConnectString();
}
@Override
@ -85,9 +83,10 @@ protected String getDropTableStatement(String tableName) {
public void setUp() {
super.setUp();
SqoopOptions options = new SqoopOptions(MySQLTestUtils.CONNECT_STRING,
SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(),
getTableName());
options.setUsername(MySQLTestUtils.getCurrentUser());
options.setUsername(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(options);
this.manager = new DirectMySQLManager(options);
try {
@ -122,26 +121,14 @@ public void tearDown() {
@Override
protected String [] getCodeGenArgv(String... extraArgs) {
String [] moreArgs = new String[extraArgs.length + 2];
int i = 0;
for (i = 0; i < extraArgs.length; i++) {
moreArgs[i] = extraArgs[i];
}
// Add username argument for mysql.
moreArgs[i++] = "--username";
moreArgs[i++] = MySQLTestUtils.getCurrentUser();
return super.getCodeGenArgv(moreArgs);
return super.getCodeGenArgv(mySQLTestUtils.addUserNameAndPasswordToArgs(extraArgs));
}
@Override
protected String [] getArgv(boolean includeHadoopFlags,
int rowsPerStatement, int statementsPerTx, String... additionalArgv) {
String [] subArgv = newStrArray(additionalArgv, "--direct",
"--username", MySQLTestUtils.getCurrentUser());
String [] subArgv = newStrArray(mySQLTestUtils.addUserNameAndPasswordToArgs(additionalArgv),"--direct");
return super.getArgv(includeHadoopFlags, rowsPerStatement,
statementsPerTx, subArgv);
}

23
src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
View File

@ -71,6 +71,7 @@ public class DirectMySQLTest extends ImportJobTestCase {
// instance variables populated during setUp, used during tests
private DirectMySQLManager manager;
private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
@Override
protected String getTablePrefix() {
@ -81,12 +82,13 @@ protected String getTablePrefix() {
public void setUp() {
super.setUp();
SqoopOptions options = new SqoopOptions(MySQLTestUtils.CONNECT_STRING,
SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(),
getTableName());
options.setUsername(MySQLTestUtils.getCurrentUser());
options.setUsername(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(options);
LOG.debug("Setting up another DirectMySQLTest: "
+ MySQLTestUtils.CONNECT_STRING);
+ mySQLTestUtils.getMySqlConnectString());
manager = new DirectMySQLManager(options);
@ -145,12 +147,13 @@ public void tearDown() {
args.add("--warehouse-dir");
args.add(getWarehouseDir());
args.add("--connect");
args.add(MySQLTestUtils.CONNECT_STRING);
args.add(mySQLTestUtils.getMySqlConnectString());
if (isDirect) {
args.add("--direct");
}
args.add("--username");
args.add(MySQLTestUtils.getCurrentUser());
args.add(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(args);
args.add("--where");
args.add("id > 1");
args.add("--num-mappers");
@ -303,9 +306,10 @@ public void testJdbcEscapedTableName() throws Exception {
// Test a JDBC-based import of a table whose name is
// a reserved sql keyword (and is thus `quoted`)
final String RESERVED_TABLE_NAME = "TABLE";
SqoopOptions options = new SqoopOptions(MySQLTestUtils.CONNECT_STRING,
SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(),
RESERVED_TABLE_NAME);
options.setUsername(MySQLTestUtils.getCurrentUser());
options.setUsername(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(options);
ConnManager mgr = new MySQLManager(options);
Connection connection = null;
@ -355,9 +359,10 @@ public void testJdbcEscapedColumnName() throws Exception {
// Test a JDBC-based import of a table with a column whose name is
// a reserved sql keyword (and is thus `quoted`).
final String TABLE_NAME = "mysql_escaped_col_table";
SqoopOptions options = new SqoopOptions(MySQLTestUtils.CONNECT_STRING,
SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(),
TABLE_NAME);
options.setUsername(MySQLTestUtils.getCurrentUser());
options.setUsername(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(options);
ConnManager mgr = new MySQLManager(options);
Connection connection = null;

24
src/test/com/cloudera/sqoop/manager/JdbcMySQLExportTest.java
View File

@ -47,6 +47,7 @@ public class JdbcMySQLExportTest extends TestExport {
// instance variables populated during setUp, used during tests.
private MySQLManager manager;
private Connection conn;
private MySQLTestUtils mySqlTestUtils = new MySQLTestUtils();
@Override
protected Connection getConnection() {
@ -66,7 +67,7 @@ protected boolean useHsqldbTestServer() {
@Override
protected String getConnectString() {
return MySQLTestUtils.CONNECT_STRING;
return mySqlTestUtils.getMySqlConnectString();
}
@Override
@ -83,9 +84,10 @@ protected String getDropTableStatement(String tableName) {
public void setUp() {
super.setUp();
SqoopOptions options = new SqoopOptions(MySQLTestUtils.CONNECT_STRING,
SqoopOptions options = new SqoopOptions(mySqlTestUtils.getMySqlConnectString(),
getTableName());
options.setUsername(MySQLTestUtils.getCurrentUser());
options.setUsername(mySqlTestUtils.getUserName());
mySqlTestUtils.addPasswordIfIsSet(options);
this.manager = new MySQLManager(options);
try {
this.conn = manager.getConnection();
@ -119,26 +121,14 @@ public void tearDown() {
@Override
protected String [] getCodeGenArgv(String... extraArgs) {
String [] moreArgs = new String[extraArgs.length + 2];
int i = 0;
for (i = 0; i < extraArgs.length; i++) {
moreArgs[i] = extraArgs[i];
}
// Add username argument for mysql.
moreArgs[i++] = "--username";
moreArgs[i++] = MySQLTestUtils.getCurrentUser();
return super.getCodeGenArgv(moreArgs);
return super.getCodeGenArgv(mySqlTestUtils.addUserNameAndPasswordToArgs(extraArgs));
}
@Override
protected String [] getArgv(boolean includeHadoopFlags,
int rowsPerStatement, int statementsPerTx, String... additionalArgv) {
String [] subArgv = newStrArray(additionalArgv,
"--username", MySQLTestUtils.getCurrentUser());
String [] subArgv = newStrArray(mySqlTestUtils.addUserNameAndPasswordToArgs(additionalArgv));
return super.getArgv(includeHadoopFlags, rowsPerStatement,
statementsPerTx, subArgv);
}

7
src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java
View File

@ -33,6 +33,8 @@
*/
public class MySQLAllTablesTest extends TestAllTables {
private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
@Override
protected boolean useHsqldbTestServer() {
return false;
@ -40,13 +42,14 @@ protected boolean useHsqldbTestServer() {
@Override
protected String getConnectString() {
return MySQLTestUtils.CONNECT_STRING;
return mySQLTestUtils.getMySqlConnectString();
}
@Override
protected SqoopOptions getSqoopOptions(Configuration conf) {
SqoopOptions opts = new SqoopOptions(conf);
opts.setUsername(MySQLTestUtils.getCurrentUser());
opts.setUsername(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(opts);
return opts;
}

6
src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java
View File

@ -39,6 +39,7 @@ public class MySQLCompatTest extends ManagerCompatTestCase {
public static final Log LOG = LogFactory.getLog(
MySQLCompatTest.class.getName());
private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
@Override
protected Log getLogger() {
@ -52,13 +53,14 @@ protected String getDbFriendlyName() {
@Override
protected String getConnectString() {
return MySQLTestUtils.CONNECT_STRING;
return mySQLTestUtils.getMySqlConnectString();
}
@Override
protected SqoopOptions getSqoopOptions(Configuration conf) {
SqoopOptions opts = new SqoopOptions(conf);
opts.setUsername(MySQLTestUtils.getCurrentUser());
opts.setUsername(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(opts);
return opts;
}

6
src/test/com/cloudera/sqoop/manager/MySQLFreeFormQueryTest.java
View File

@ -37,6 +37,7 @@ public class MySQLFreeFormQueryTest extends TestFreeFormQueryImport {
public static final Log LOG = LogFactory.getLog(
MySQLFreeFormQueryTest.class.getName());
private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
@Override
protected Log getLogger() {
@ -50,13 +51,14 @@ protected boolean useHsqldbTestServer() {
@Override
protected String getConnectString() {
return MySQLTestUtils.CONNECT_STRING;
return mySQLTestUtils.getMySqlConnectString();
}
@Override
protected SqoopOptions getSqoopOptions(Configuration conf) {
SqoopOptions opts = new SqoopOptions(conf);
opts.setUsername(MySQLTestUtils.getCurrentUser());
opts.setUsername(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(opts);
return opts;
}

6
src/test/com/cloudera/sqoop/manager/MySQLLobAvroImportTest.java
View File

@ -37,6 +37,7 @@ public class MySQLLobAvroImportTest extends LobAvroImportTestCase {
public static final Log LOG = LogFactory.getLog(
OracleCompatTest.class.getName());
private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
@Override
protected Log getLogger() {
@ -50,13 +51,14 @@ protected String getDbFriendlyName() {
@Override
protected String getConnectString() {
return MySQLTestUtils.CONNECT_STRING;
return mySQLTestUtils.getMySqlConnectString();
}
@Override
protected SqoopOptions getSqoopOptions(Configuration conf) {
SqoopOptions opts = new SqoopOptions(conf);
opts.setUsername(MySQLTestUtils.getCurrentUser());
opts.setUsername(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(opts);
return opts;
}

92
src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java
View File

@ -18,13 +18,16 @@
package com.cloudera.sqoop.manager;
import com.cloudera.sqoop.SqoopOptions;
import jodd.util.StringUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.util.ArrayList;
/**
* Utilities for mysql-based tests.
@ -34,29 +37,72 @@ public final class MySQLTestUtils {
public static final Log LOG = LogFactory.getLog(
MySQLTestUtils.class.getName());
public static final String HOST_URL = System.getProperty(
private String hostUrl;
private String userName;
private String userPass;
private String mysqlDbNAme;
private String mySqlConnectString;
public MySQLTestUtils() {
hostUrl = System.getProperty(
"sqoop.test.mysql.connectstring.host_url",
"jdbc:mysql://localhost/");
userName = System.getProperty("sqoop.test.mysql.username", getCurrentUser());
userPass = System.getProperty("sqoop.test.mysql.password");
public static final String USER_NAME = System.getProperty("sqoop.test.mysql.username", "sqoop");
public static final String USER_PASS = System.getProperty("sqoop.test.mysql.password", "sqoop");
mysqlDbNAme = System.getProperty("sqoop.test.mysql.databasename", "sqooptestdb");
mySqlConnectString = getHostUrl() + getMysqlDbNAme();
}
public static final String MYSQL_DATABASE_NAME = System.getProperty("sqoop.test.mysql.databasename", "sqooptestdb");
public static final String TABLE_NAME = "EMPLOYEES_MYSQL";
public static final String CONNECT_STRING = HOST_URL + MYSQL_DATABASE_NAME;
public String getHostUrl() {
return hostUrl;
}
private MySQLTestUtils() { }
public String getUserName() {
return userName;
}
public String getUserPass() {
return userPass;
}
public String getMysqlDbNAme() {
return mysqlDbNAme;
}
public String getMySqlConnectString() {
return mySqlConnectString;
}
public String[] addUserNameAndPasswordToArgs(String[] extraArgs) {
int extraLength = isSet(getUserPass()) ? 4 : 2;
String[] moreArgs = new String[extraArgs.length + extraLength];
int i = 0;
for (i = 0; i < extraArgs.length; i++) {
moreArgs[i] = extraArgs[i];
}
// Add username argument for mysql.
moreArgs[i++] = "--username";
moreArgs[i++] = getUserName();
if (isSet(userPass)) {
moreArgs[i++] = "--password";
moreArgs[i++] = getUserPass();
}
return moreArgs;
}
/** @return the current username. */
public static String getCurrentUser() {
// First, check the $USER environment variable.
String envUser = System.getenv("USER");
if (null != envUser) {
return envUser;
}
// Try `whoami`
String [] whoamiArgs = new String[1];
String[] whoamiArgs = new String[1];
whoamiArgs[0] = "whoami";
Process p = null;
BufferedReader r = null;
@ -78,7 +124,6 @@ public static String getCurrentUser() {
+ ioe.toString());
}
}
// wait for whoami to exit.
while (p != null) {
try {
@ -92,6 +137,25 @@ public static String getCurrentUser() {
continue; // loop around.
}
}
}
}
public void addPasswordIfIsSet(ArrayList<String> args) {
if (isSet(userPass)) {
args.add("--password");
args.add(getUserPass());
}
}
private boolean isSet(String userPass) {
return !StringUtil.isBlank(userPass);
}
public void addPasswordIfIsSet(SqoopOptions opts) {
if (isSet(userPass)) {
opts.setPassword(getUserPass());
}
}
}

6
src/test/org/apache/sqoop/manager/mysql/MySqlCallExportTest.java
View File

@ -49,6 +49,7 @@ public class MySqlCallExportTest extends ExportJobTestCase {
private final String tableName = "MYSQL_CALL_EXPORT_BASE_TABLE";
private final String procName = "MYSQL_CALL_EXPORT_PROC";
private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
@Override
public void setUp() {
@ -150,13 +151,14 @@ protected boolean useHsqldbTestServer() {
@Override
protected String getConnectString() {
return MySQLTestUtils.CONNECT_STRING;
return mySQLTestUtils.getMySqlConnectString();
}
@Override
protected SqoopOptions getSqoopOptions(Configuration conf) {
SqoopOptions opts = new SqoopOptions(conf);
opts.setUsername(MySQLTestUtils.getCurrentUser());
opts.setUsername(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(opts);
return opts;
}

12
src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java
View File

@ -38,6 +38,7 @@ public class MySqlColumnEscapeImportTest extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog(
MySqlColumnEscapeImportTest.class.getName());
private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
@Override
protected boolean useHsqldbTestServer() {
@ -46,14 +47,14 @@ protected boolean useHsqldbTestServer() {
@Override
protected String getConnectString() {
return MySQLTestUtils.CONNECT_STRING;
return mySQLTestUtils.getMySqlConnectString();
}
@Override
protected SqoopOptions getSqoopOptions(Configuration conf) {
SqoopOptions opts = new SqoopOptions(conf);
opts.setUsername(MySQLTestUtils.USER_NAME);
opts.setPassword(MySQLTestUtils.USER_PASS);
opts.setUsername(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(opts);
return opts;
}
@ -80,9 +81,8 @@ public void tearDown() {
args.add("--connect");
args.add(getConnectString());
args.add("--username");
args.add(MySQLTestUtils.USER_NAME);
args.add("--password");
args.add(MySQLTestUtils.USER_PASS);
args.add(mySQLTestUtils.getUserName());
mySQLTestUtils.addPasswordIfIsSet(args);
args.add("--target-dir");
args.add(getWarehouseDir());
args.add("--num-mappers");