From 0f13c474bf91f6609b9bdd7a1aa25b250f07e398 Mon Sep 17 00:00:00 2001 From: Attila Szabo Date: Fri, 7 Oct 2016 01:57:12 +0200 Subject: [PATCH] SQOOP-3021: ClassWriter fails if a column name contains a backslash character (Szabolcs Vasas via Attila Szabo) --- .../org/apache/sqoop/orm/ClassWriter.java | 3 +- .../com/cloudera/sqoop/ThirdPartyTests.java | 4 + .../sqoop/manager/MySQLTestUtils.java | 5 +- .../sqoop/testutil/BaseSqoopTestCase.java | 11 +- .../mysql/MySqlColumnEscapeImportTest.java | 112 ++++++++++++++++++ .../oracle/OracleColumnEscapeImportTest.java | 110 +++++++++++++++++ 6 files changed, 239 insertions(+), 6 deletions(-) create mode 100644 src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java create mode 100644 src/test/org/apache/sqoop/manager/oracle/OracleColumnEscapeImportTest.java diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java index 9d91887d..2f14b6ec 100644 --- a/src/java/org/apache/sqoop/orm/ClassWriter.java +++ b/src/java/org/apache/sqoop/orm/ClassWriter.java @@ -31,6 +31,7 @@ import java.util.Properties; import java.util.Set; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.BytesWritable; @@ -1118,7 +1119,7 @@ private void generateConstructorAndInitMethods(Map colTypes, St * @return */ private String serializeRawColName(String name) { - return name.replace("\"", "\\\""); + return StringEscapeUtils.escapeJava(name); } /** diff --git a/src/test/com/cloudera/sqoop/ThirdPartyTests.java b/src/test/com/cloudera/sqoop/ThirdPartyTests.java index 3103bd4d..7e10c687 100644 --- a/src/test/com/cloudera/sqoop/ThirdPartyTests.java +++ b/src/test/com/cloudera/sqoop/ThirdPartyTests.java @@ -50,6 +50,7 @@ import org.apache.sqoop.manager.cubrid.CubridAuthTest; import org.apache.sqoop.manager.cubrid.CubridCompatTest; import org.apache.sqoop.manager.mysql.MySqlCallExportTest; +import org.apache.sqoop.manager.mysql.MySqlColumnEscapeImportTest; import org.apache.sqoop.manager.netezza.DirectNetezzaExportManualTest; import org.apache.sqoop.manager.netezza.DirectNetezzaHCatExportManualTest; import org.apache.sqoop.manager.netezza.DirectNetezzaHCatImportManualTest; @@ -57,6 +58,7 @@ import org.apache.sqoop.manager.netezza.NetezzaImportManualTest; import org.apache.sqoop.manager.oracle.OraOopDataDrivenDBInputFormatConnectionCloseTest; import org.apache.sqoop.manager.oracle.OracleCallExportTest; +import org.apache.sqoop.manager.oracle.OracleColumnEscapeImportTest; import org.apache.sqoop.manager.oracle.OracleIncrementalImportTest; import org.apache.sqoop.manager.oracle.OracleSplitterTest; import org.apache.sqoop.manager.sqlserver.SQLServerDatatypeExportDelimitedFileManualTest; @@ -92,6 +94,7 @@ public static Test suite() { suite.addTestSuite(JdbcMySQLExportTest.class); suite.addTestSuite(MySQLAuthTest.class); suite.addTestSuite(MySQLCompatTest.class); + suite.addTestSuite(MySqlColumnEscapeImportTest.class); // Oracle suite.addTestSuite(OracleExportTest.class); @@ -100,6 +103,7 @@ public static Test suite() { suite.addTestSuite(OracleIncrementalImportTest.class); suite.addTestSuite(OracleSplitterTest.class); suite.addTestSuite(OraOopDataDrivenDBInputFormatConnectionCloseTest.class); + suite.addTestSuite(OracleColumnEscapeImportTest.class); // SQL Server suite.addTestSuite(SQLServerDatatypeExportDelimitedFileManualTest.class); diff --git a/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java b/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java index b5b9b6ee..0af79d69 100644 --- a/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java +++ b/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java @@ -38,7 +38,10 @@ public final class MySQLTestUtils { "sqoop.test.mysql.connectstring.host_url", "jdbc:mysql://localhost/"); - public static final String MYSQL_DATABASE_NAME = "sqooptestdb"; + public static final String USER_NAME = System.getProperty("sqoop.test.mysql.username", "sqoop"); + public static final String USER_PASS = System.getProperty("sqoop.test.mysql.password", "sqoop"); + + public static final String MYSQL_DATABASE_NAME = System.getProperty("sqoop.test.mysql.databasename", "sqooptestdb"); public static final String TABLE_NAME = "EMPLOYEES_MYSQL"; public static final String CONNECT_STRING = HOST_URL + MYSQL_DATABASE_NAME; diff --git a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java index 8dddd026..f8021beb 100644 --- a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java +++ b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java @@ -295,8 +295,7 @@ protected String getColName(int i) { */ protected void dropTableIfExists(String table) throws SQLException { Connection conn = getManager().getConnection(); - PreparedStatement statement = conn.prepareStatement( - "DROP TABLE " + manager.escapeTableName(table) + " IF EXISTS", + PreparedStatement statement = conn.prepareStatement(dropTableIfExistsCommand(table), ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); try { statement.executeUpdate(); @@ -306,6 +305,10 @@ protected void dropTableIfExists(String table) throws SQLException { } } + protected String dropTableIfExistsCommand(String table) { + return "DROP TABLE " + manager.escapeTableName(table) + " IF EXISTS"; + } + /** * Create a table with a set of columns with their names and add a row of values. * @param colNames Column names @@ -331,7 +334,7 @@ protected void createTableWithColTypesAndNames(String[] colNames, conn = getManager().getConnection(); for (int i = 0; i < colTypes.length; i++) { - columnDefStr += '"' + colNames[i].toUpperCase() + '"' + " " + colTypes[i]; + columnDefStr += manager.escapeColName(colNames[i].toUpperCase()) + " " + colTypes[i]; if (i < colTypes.length - 1) { columnDefStr += ", "; } @@ -363,7 +366,7 @@ protected void createTableWithColTypesAndNames(String[] colNames, String columnListStr = ""; String valueListStr = ""; for (int i = 0; i < colTypes.length; i++) { - columnListStr += '"' + colNames[i].toUpperCase() + '"'; + columnListStr += manager.escapeColName(colNames[i].toUpperCase()); valueListStr += vals[count * colTypes.length + i]; if (i < colTypes.length - 1) { columnListStr += ", "; diff --git a/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java b/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java new file mode 100644 index 00000000..87cd3896 --- /dev/null +++ b/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.mysql; + +import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.manager.MySQLTestUtils; +import com.cloudera.sqoop.testutil.CommonArgs; +import com.cloudera.sqoop.testutil.ImportJobTestCase; +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; + +public class MySqlColumnEscapeImportTest extends ImportJobTestCase { + + public static final Log LOG = LogFactory.getLog( + MySqlColumnEscapeImportTest.class.getName()); + + @Override + protected boolean useHsqldbTestServer() { + return false; + } + + @Override + protected String getConnectString() { + return MySQLTestUtils.CONNECT_STRING; + } + + @Override + protected SqoopOptions getSqoopOptions(Configuration conf) { + SqoopOptions opts = new SqoopOptions(conf); + opts.setUsername(MySQLTestUtils.USER_NAME); + opts.setPassword(MySQLTestUtils.USER_PASS); + return opts; + } + + @Override + protected String dropTableIfExistsCommand(String table) { + return "DROP TABLE IF EXISTS " + getManager().escapeTableName(table); + } + + @Override + public void tearDown() { + try { + dropTableIfExists(getTableName()); + } catch (SQLException e) { + LOG.error("Could not delete test table", e); + } + super.tearDown(); + } + + protected String [] getArgv() { + ArrayList args = new ArrayList(); + + CommonArgs.addHadoopFlags(args); + + args.add("--connect"); + args.add(getConnectString()); + args.add("--username"); + args.add(MySQLTestUtils.USER_NAME); + args.add("--password"); + args.add(MySQLTestUtils.USER_PASS); + args.add("--target-dir"); + args.add(getWarehouseDir()); + args.add("--num-mappers"); + args.add("1"); + args.add("--table"); + args.add(getTableName()); + + return args.toArray(new String[0]); + } + + public void testEscapeColumnWithDoubleQuote() throws IOException { + String[] colNames = { "column\"withdoublequote" }; + String[] types = { "VARCHAR(50)"}; + String[] vals = { "'hello, world'"}; + createTableWithColTypesAndNames(colNames, types, vals); + String[] args = getArgv(); + runImport(args); + + Path warehousePath = new Path(this.getWarehouseDir()); + Path filePath = new Path(warehousePath, "part-m-00000"); + String output = Files.toString(new File(filePath.toString()), Charsets.UTF_8); + + assertEquals("hello, world", output.trim()); + } + +} + diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleColumnEscapeImportTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleColumnEscapeImportTest.java new file mode 100644 index 00000000..d428040d --- /dev/null +++ b/src/test/org/apache/sqoop/manager/oracle/OracleColumnEscapeImportTest.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.oracle; + +import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.manager.OracleUtils; +import com.cloudera.sqoop.testutil.CommonArgs; +import com.cloudera.sqoop.testutil.ImportJobTestCase; +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; + +public class OracleColumnEscapeImportTest extends ImportJobTestCase { + + public static final Log LOG = LogFactory.getLog( + OracleColumnEscapeImportTest.class.getName()); + + @Override + protected boolean useHsqldbTestServer() { + return false; + } + + @Override + protected String getConnectString() { + return OracleUtils.CONNECT_STRING; + } + + @Override + protected SqoopOptions getSqoopOptions(Configuration conf) { + SqoopOptions opts = new SqoopOptions(conf); + OracleUtils.setOracleAuth(opts); + return opts; + } + + @Override + protected void dropTableIfExists(String table) throws SQLException { + OracleUtils.dropTable(table, getManager()); + } + + @Override + public void tearDown() { + try { + OracleUtils.dropTable(getTableName(), getManager()); + } catch (SQLException e) { + LOG.error("Test table could not be dropped", e); + } + super.tearDown(); + } + + protected String [] getArgv() { + ArrayList args = new ArrayList(); + + CommonArgs.addHadoopFlags(args); + + args.add("--connect"); + args.add(getConnectString()); + args.add("--username"); + args.add(OracleUtils.ORACLE_USER_NAME); + args.add("--password"); + args.add(OracleUtils.ORACLE_USER_PASS); + args.add("--target-dir"); + args.add(getWarehouseDir()); + args.add("--num-mappers"); + args.add("1"); + args.add("--query"); + args.add("select REGEXP_REPLACE(TRIM(" + getColName(0) + "), '\\:','!') from " + getTableName() + " WHERE $CONDITIONS"); + + return args.toArray(new String[0]); + } + + public void testRegexpReplaceEscapeWithSpecialCharacters() throws IOException { + String [] types = { "VARCHAR(50)"}; + String [] vals = { "'hello, world:'"}; + createTableWithColTypes(types, vals); + String[] args = getArgv(); + runImport(args); + + Path warehousePath = new Path(this.getWarehouseDir()); + Path filePath = new Path(warehousePath, "part-m-00000"); + String output = Files.toString(new File(filePath.toString()), Charsets.UTF_8); + + assertEquals("hello, world!", output.trim()); + } + +} +