From 48cecdc99e45514adff5b9ea8112f5fb366aee14 Mon Sep 17 00:00:00 2001 From: Attila Szabo Date: Tue, 29 Nov 2016 15:33:16 +0100 Subject: [PATCH] SQOOP-3055: Fixing MySQL tests failing due to ignored test inputs/configuration (Anna Szonyi via Attila Szabo) --- build.xml | 15 +++ .../sqoop/manager/DirectMySQLExportTest.java | 27 ++--- .../sqoop/manager/DirectMySQLTest.java | 23 +++-- .../sqoop/manager/JdbcMySQLExportTest.java | 24 ++--- .../sqoop/manager/MySQLAllTablesTest.java | 7 +- .../sqoop/manager/MySQLCompatTest.java | 6 +- .../sqoop/manager/MySQLFreeFormQueryTest.java | 6 +- .../sqoop/manager/MySQLLobAvroImportTest.java | 6 +- .../sqoop/manager/MySQLTestUtils.java | 98 +++++++++++++++---- .../manager/mysql/MySqlCallExportTest.java | 6 +- .../mysql/MySqlColumnEscapeImportTest.java | 12 +-- 11 files changed, 151 insertions(+), 79 deletions(-) diff --git a/build.xml b/build.xml index 6e8d3939..cc8760b2 100644 --- a/build.xml +++ b/build.xml @@ -372,6 +372,12 @@ + + + + @@ -983,6 +989,15 @@ + + + + + + diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java index 79e90e24..7f086752 100644 --- a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java +++ b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java @@ -39,9 +39,6 @@ import com.cloudera.sqoop.TestExport; import com.cloudera.sqoop.mapreduce.MySQLExportMapper; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - /** * Test the DirectMySQLManager implementation's exportJob() functionality. */ @@ -55,6 +52,7 @@ public class DirectMySQLExportTest extends TestExport { // instance variables populated during setUp, used during tests. private DirectMySQLManager manager; private Connection conn; + private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); @Override protected Connection getConnection() { @@ -68,7 +66,7 @@ protected boolean useHsqldbTestServer() { @Override protected String getConnectString() { - return MySQLTestUtils.CONNECT_STRING; + return mySQLTestUtils.getMySqlConnectString(); } @Override @@ -85,9 +83,10 @@ protected String getDropTableStatement(String tableName) { public void setUp() { super.setUp(); - SqoopOptions options = new SqoopOptions(MySQLTestUtils.CONNECT_STRING, + SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), getTableName()); - options.setUsername(MySQLTestUtils.getCurrentUser()); + options.setUsername(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(options); this.manager = new DirectMySQLManager(options); try { @@ -122,26 +121,14 @@ public void tearDown() { @Override protected String [] getCodeGenArgv(String... extraArgs) { - - String [] moreArgs = new String[extraArgs.length + 2]; - int i = 0; - for (i = 0; i < extraArgs.length; i++) { - moreArgs[i] = extraArgs[i]; - } - - // Add username argument for mysql. - moreArgs[i++] = "--username"; - moreArgs[i++] = MySQLTestUtils.getCurrentUser(); - - return super.getCodeGenArgv(moreArgs); + return super.getCodeGenArgv(mySQLTestUtils.addUserNameAndPasswordToArgs(extraArgs)); } @Override protected String [] getArgv(boolean includeHadoopFlags, int rowsPerStatement, int statementsPerTx, String... additionalArgv) { - String [] subArgv = newStrArray(additionalArgv, "--direct", - "--username", MySQLTestUtils.getCurrentUser()); + String [] subArgv = newStrArray(mySQLTestUtils.addUserNameAndPasswordToArgs(additionalArgv),"--direct"); return super.getArgv(includeHadoopFlags, rowsPerStatement, statementsPerTx, subArgv); } diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java index d0f0c502..faa0a3e1 100644 --- a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java +++ b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java @@ -71,6 +71,7 @@ public class DirectMySQLTest extends ImportJobTestCase { // instance variables populated during setUp, used during tests private DirectMySQLManager manager; + private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); @Override protected String getTablePrefix() { @@ -81,12 +82,13 @@ protected String getTablePrefix() { public void setUp() { super.setUp(); - SqoopOptions options = new SqoopOptions(MySQLTestUtils.CONNECT_STRING, + SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), getTableName()); - options.setUsername(MySQLTestUtils.getCurrentUser()); + options.setUsername(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(options); LOG.debug("Setting up another DirectMySQLTest: " - + MySQLTestUtils.CONNECT_STRING); + + mySQLTestUtils.getMySqlConnectString()); manager = new DirectMySQLManager(options); @@ -145,12 +147,13 @@ public void tearDown() { args.add("--warehouse-dir"); args.add(getWarehouseDir()); args.add("--connect"); - args.add(MySQLTestUtils.CONNECT_STRING); + args.add(mySQLTestUtils.getMySqlConnectString()); if (isDirect) { args.add("--direct"); } args.add("--username"); - args.add(MySQLTestUtils.getCurrentUser()); + args.add(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(args); args.add("--where"); args.add("id > 1"); args.add("--num-mappers"); @@ -303,9 +306,10 @@ public void testJdbcEscapedTableName() throws Exception { // Test a JDBC-based import of a table whose name is // a reserved sql keyword (and is thus `quoted`) final String RESERVED_TABLE_NAME = "TABLE"; - SqoopOptions options = new SqoopOptions(MySQLTestUtils.CONNECT_STRING, + SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), RESERVED_TABLE_NAME); - options.setUsername(MySQLTestUtils.getCurrentUser()); + options.setUsername(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(options); ConnManager mgr = new MySQLManager(options); Connection connection = null; @@ -355,9 +359,10 @@ public void testJdbcEscapedColumnName() throws Exception { // Test a JDBC-based import of a table with a column whose name is // a reserved sql keyword (and is thus `quoted`). final String TABLE_NAME = "mysql_escaped_col_table"; - SqoopOptions options = new SqoopOptions(MySQLTestUtils.CONNECT_STRING, + SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), TABLE_NAME); - options.setUsername(MySQLTestUtils.getCurrentUser()); + options.setUsername(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(options); ConnManager mgr = new MySQLManager(options); Connection connection = null; diff --git a/src/test/com/cloudera/sqoop/manager/JdbcMySQLExportTest.java b/src/test/com/cloudera/sqoop/manager/JdbcMySQLExportTest.java index 86b9aab6..5c6e5671 100644 --- a/src/test/com/cloudera/sqoop/manager/JdbcMySQLExportTest.java +++ b/src/test/com/cloudera/sqoop/manager/JdbcMySQLExportTest.java @@ -47,6 +47,7 @@ public class JdbcMySQLExportTest extends TestExport { // instance variables populated during setUp, used during tests. private MySQLManager manager; private Connection conn; + private MySQLTestUtils mySqlTestUtils = new MySQLTestUtils(); @Override protected Connection getConnection() { @@ -66,7 +67,7 @@ protected boolean useHsqldbTestServer() { @Override protected String getConnectString() { - return MySQLTestUtils.CONNECT_STRING; + return mySqlTestUtils.getMySqlConnectString(); } @Override @@ -83,9 +84,10 @@ protected String getDropTableStatement(String tableName) { public void setUp() { super.setUp(); - SqoopOptions options = new SqoopOptions(MySQLTestUtils.CONNECT_STRING, + SqoopOptions options = new SqoopOptions(mySqlTestUtils.getMySqlConnectString(), getTableName()); - options.setUsername(MySQLTestUtils.getCurrentUser()); + options.setUsername(mySqlTestUtils.getUserName()); + mySqlTestUtils.addPasswordIfIsSet(options); this.manager = new MySQLManager(options); try { this.conn = manager.getConnection(); @@ -119,26 +121,14 @@ public void tearDown() { @Override protected String [] getCodeGenArgv(String... extraArgs) { - - String [] moreArgs = new String[extraArgs.length + 2]; - int i = 0; - for (i = 0; i < extraArgs.length; i++) { - moreArgs[i] = extraArgs[i]; - } - - // Add username argument for mysql. - moreArgs[i++] = "--username"; - moreArgs[i++] = MySQLTestUtils.getCurrentUser(); - - return super.getCodeGenArgv(moreArgs); + return super.getCodeGenArgv(mySqlTestUtils.addUserNameAndPasswordToArgs(extraArgs)); } @Override protected String [] getArgv(boolean includeHadoopFlags, int rowsPerStatement, int statementsPerTx, String... additionalArgv) { - String [] subArgv = newStrArray(additionalArgv, - "--username", MySQLTestUtils.getCurrentUser()); + String [] subArgv = newStrArray(mySqlTestUtils.addUserNameAndPasswordToArgs(additionalArgv)); return super.getArgv(includeHadoopFlags, rowsPerStatement, statementsPerTx, subArgv); } diff --git a/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java b/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java index bad3c294..ce4af814 100644 --- a/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java +++ b/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java @@ -33,6 +33,8 @@ */ public class MySQLAllTablesTest extends TestAllTables { + private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); + @Override protected boolean useHsqldbTestServer() { return false; @@ -40,13 +42,14 @@ protected boolean useHsqldbTestServer() { @Override protected String getConnectString() { - return MySQLTestUtils.CONNECT_STRING; + return mySQLTestUtils.getMySqlConnectString(); } @Override protected SqoopOptions getSqoopOptions(Configuration conf) { SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(MySQLTestUtils.getCurrentUser()); + opts.setUsername(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(opts); return opts; } diff --git a/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java b/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java index 675b6b84..65399721 100644 --- a/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java +++ b/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java @@ -39,6 +39,7 @@ public class MySQLCompatTest extends ManagerCompatTestCase { public static final Log LOG = LogFactory.getLog( MySQLCompatTest.class.getName()); + private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); @Override protected Log getLogger() { @@ -52,13 +53,14 @@ protected String getDbFriendlyName() { @Override protected String getConnectString() { - return MySQLTestUtils.CONNECT_STRING; + return mySQLTestUtils.getMySqlConnectString(); } @Override protected SqoopOptions getSqoopOptions(Configuration conf) { SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(MySQLTestUtils.getCurrentUser()); + opts.setUsername(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(opts); return opts; } diff --git a/src/test/com/cloudera/sqoop/manager/MySQLFreeFormQueryTest.java b/src/test/com/cloudera/sqoop/manager/MySQLFreeFormQueryTest.java index 935d948b..22547f2d 100644 --- a/src/test/com/cloudera/sqoop/manager/MySQLFreeFormQueryTest.java +++ b/src/test/com/cloudera/sqoop/manager/MySQLFreeFormQueryTest.java @@ -37,6 +37,7 @@ public class MySQLFreeFormQueryTest extends TestFreeFormQueryImport { public static final Log LOG = LogFactory.getLog( MySQLFreeFormQueryTest.class.getName()); + private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); @Override protected Log getLogger() { @@ -50,13 +51,14 @@ protected boolean useHsqldbTestServer() { @Override protected String getConnectString() { - return MySQLTestUtils.CONNECT_STRING; + return mySQLTestUtils.getMySqlConnectString(); } @Override protected SqoopOptions getSqoopOptions(Configuration conf) { SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(MySQLTestUtils.getCurrentUser()); + opts.setUsername(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(opts); return opts; } diff --git a/src/test/com/cloudera/sqoop/manager/MySQLLobAvroImportTest.java b/src/test/com/cloudera/sqoop/manager/MySQLLobAvroImportTest.java index df781420..cd146f4d 100644 --- a/src/test/com/cloudera/sqoop/manager/MySQLLobAvroImportTest.java +++ b/src/test/com/cloudera/sqoop/manager/MySQLLobAvroImportTest.java @@ -37,6 +37,7 @@ public class MySQLLobAvroImportTest extends LobAvroImportTestCase { public static final Log LOG = LogFactory.getLog( OracleCompatTest.class.getName()); + private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); @Override protected Log getLogger() { @@ -50,13 +51,14 @@ protected String getDbFriendlyName() { @Override protected String getConnectString() { - return MySQLTestUtils.CONNECT_STRING; + return mySQLTestUtils.getMySqlConnectString(); } @Override protected SqoopOptions getSqoopOptions(Configuration conf) { SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(MySQLTestUtils.getCurrentUser()); + opts.setUsername(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(opts); return opts; } diff --git a/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java b/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java index 0af79d69..2e2448a4 100644 --- a/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java +++ b/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java @@ -18,13 +18,16 @@ package com.cloudera.sqoop.manager; +import com.cloudera.sqoop.SqoopOptions; +import jodd.util.StringUtil; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import java.util.ArrayList; /** * Utilities for mysql-based tests. @@ -34,29 +37,72 @@ public final class MySQLTestUtils { public static final Log LOG = LogFactory.getLog( MySQLTestUtils.class.getName()); - public static final String HOST_URL = System.getProperty( - "sqoop.test.mysql.connectstring.host_url", - "jdbc:mysql://localhost/"); + private String hostUrl; - public static final String USER_NAME = System.getProperty("sqoop.test.mysql.username", "sqoop"); - public static final String USER_PASS = System.getProperty("sqoop.test.mysql.password", "sqoop"); + private String userName; + private String userPass; - public static final String MYSQL_DATABASE_NAME = System.getProperty("sqoop.test.mysql.databasename", "sqooptestdb"); - public static final String TABLE_NAME = "EMPLOYEES_MYSQL"; - public static final String CONNECT_STRING = HOST_URL + MYSQL_DATABASE_NAME; + private String mysqlDbNAme; + private String mySqlConnectString; - private MySQLTestUtils() { } + public MySQLTestUtils() { + hostUrl = System.getProperty( + "sqoop.test.mysql.connectstring.host_url", + "jdbc:mysql://localhost/"); + userName = System.getProperty("sqoop.test.mysql.username", getCurrentUser()); + userPass = System.getProperty("sqoop.test.mysql.password"); + + mysqlDbNAme = System.getProperty("sqoop.test.mysql.databasename", "sqooptestdb"); + mySqlConnectString = getHostUrl() + getMysqlDbNAme(); + } + + public String getHostUrl() { + return hostUrl; + } + + public String getUserName() { + return userName; + } + + public String getUserPass() { + return userPass; + } + + public String getMysqlDbNAme() { + return mysqlDbNAme; + } + + + public String getMySqlConnectString() { + return mySqlConnectString; + } + + public String[] addUserNameAndPasswordToArgs(String[] extraArgs) { + int extraLength = isSet(getUserPass()) ? 4 : 2; + String[] moreArgs = new String[extraArgs.length + extraLength]; + int i = 0; + for (i = 0; i < extraArgs.length; i++) { + moreArgs[i] = extraArgs[i]; + } + + // Add username argument for mysql. + moreArgs[i++] = "--username"; + moreArgs[i++] = getUserName(); + if (isSet(userPass)) { + moreArgs[i++] = "--password"; + moreArgs[i++] = getUserPass(); + } + return moreArgs; + } - /** @return the current username. */ public static String getCurrentUser() { // First, check the $USER environment variable. String envUser = System.getenv("USER"); if (null != envUser) { return envUser; } - // Try `whoami` - String [] whoamiArgs = new String[1]; + String[] whoamiArgs = new String[1]; whoamiArgs[0] = "whoami"; Process p = null; BufferedReader r = null; @@ -78,7 +124,6 @@ public static String getCurrentUser() { + ioe.toString()); } } - // wait for whoami to exit. while (p != null) { try { @@ -92,6 +137,25 @@ public static String getCurrentUser() { continue; // loop around. } } + } } -} + + public void addPasswordIfIsSet(ArrayList args) { + if (isSet(userPass)) { + args.add("--password"); + args.add(getUserPass()); + } + } + + private boolean isSet(String userPass) { + return !StringUtil.isBlank(userPass); + } + + public void addPasswordIfIsSet(SqoopOptions opts) { + if (isSet(userPass)) { + opts.setPassword(getUserPass()); + } + } + +} \ No newline at end of file diff --git a/src/test/org/apache/sqoop/manager/mysql/MySqlCallExportTest.java b/src/test/org/apache/sqoop/manager/mysql/MySqlCallExportTest.java index 6bf939b6..420ab19d 100644 --- a/src/test/org/apache/sqoop/manager/mysql/MySqlCallExportTest.java +++ b/src/test/org/apache/sqoop/manager/mysql/MySqlCallExportTest.java @@ -49,6 +49,7 @@ public class MySqlCallExportTest extends ExportJobTestCase { private final String tableName = "MYSQL_CALL_EXPORT_BASE_TABLE"; private final String procName = "MYSQL_CALL_EXPORT_PROC"; + private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); @Override public void setUp() { @@ -150,13 +151,14 @@ protected boolean useHsqldbTestServer() { @Override protected String getConnectString() { - return MySQLTestUtils.CONNECT_STRING; + return mySQLTestUtils.getMySqlConnectString(); } @Override protected SqoopOptions getSqoopOptions(Configuration conf) { SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(MySQLTestUtils.getCurrentUser()); + opts.setUsername(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(opts); return opts; } diff --git a/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java b/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java index 87cd3896..8a6b6fe9 100644 --- a/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java +++ b/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java @@ -38,6 +38,7 @@ public class MySqlColumnEscapeImportTest extends ImportJobTestCase { public static final Log LOG = LogFactory.getLog( MySqlColumnEscapeImportTest.class.getName()); + private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); @Override protected boolean useHsqldbTestServer() { @@ -46,14 +47,14 @@ protected boolean useHsqldbTestServer() { @Override protected String getConnectString() { - return MySQLTestUtils.CONNECT_STRING; + return mySQLTestUtils.getMySqlConnectString(); } @Override protected SqoopOptions getSqoopOptions(Configuration conf) { SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(MySQLTestUtils.USER_NAME); - opts.setPassword(MySQLTestUtils.USER_PASS); + opts.setUsername(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(opts); return opts; } @@ -80,9 +81,8 @@ public void tearDown() { args.add("--connect"); args.add(getConnectString()); args.add("--username"); - args.add(MySQLTestUtils.USER_NAME); - args.add("--password"); - args.add(MySQLTestUtils.USER_PASS); + args.add(mySQLTestUtils.getUserName()); + mySQLTestUtils.addPasswordIfIsSet(args); args.add("--target-dir"); args.add(getWarehouseDir()); args.add("--num-mappers");