github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-19 02:10:54 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3021: ClassWriter fails if a column name contains a backslash character

Browse Source 
(Szabolcs Vasas via Attila Szabo)
This commit is contained in:
Attila Szabo 2016-10-07 01:57:12 +02:00
parent 8f3989d9ad
commit 0f13c474bf
6 changed files with 239 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/java/org/apache/sqoop/orm/ClassWriter.java
View File

@ -31,6 +31,7 @@
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.BytesWritable;
@ -1118,7 +1119,7 @@ private void generateConstructorAndInitMethods(Map<String, Integer> colTypes, St
* @return
*/
private String serializeRawColName(String name) {
return name.replace("\"", "\\\"");
return StringEscapeUtils.escapeJava(name);
}
/**

4
src/test/com/cloudera/sqoop/ThirdPartyTests.java
View File

@ -50,6 +50,7 @@
import org.apache.sqoop.manager.cubrid.CubridAuthTest;
import org.apache.sqoop.manager.cubrid.CubridCompatTest;
import org.apache.sqoop.manager.mysql.MySqlCallExportTest;
import org.apache.sqoop.manager.mysql.MySqlColumnEscapeImportTest;
import org.apache.sqoop.manager.netezza.DirectNetezzaExportManualTest;
import org.apache.sqoop.manager.netezza.DirectNetezzaHCatExportManualTest;
import org.apache.sqoop.manager.netezza.DirectNetezzaHCatImportManualTest;
@ -57,6 +58,7 @@
import org.apache.sqoop.manager.netezza.NetezzaImportManualTest;
import org.apache.sqoop.manager.oracle.OraOopDataDrivenDBInputFormatConnectionCloseTest;
import org.apache.sqoop.manager.oracle.OracleCallExportTest;
import org.apache.sqoop.manager.oracle.OracleColumnEscapeImportTest;
import org.apache.sqoop.manager.oracle.OracleIncrementalImportTest;
import org.apache.sqoop.manager.oracle.OracleSplitterTest;
import org.apache.sqoop.manager.sqlserver.SQLServerDatatypeExportDelimitedFileManualTest;
@ -92,6 +94,7 @@ public static Test suite() {
suite.addTestSuite(JdbcMySQLExportTest.class);
suite.addTestSuite(MySQLAuthTest.class);
suite.addTestSuite(MySQLCompatTest.class);
suite.addTestSuite(MySqlColumnEscapeImportTest.class);
// Oracle
suite.addTestSuite(OracleExportTest.class);
@ -100,6 +103,7 @@ public static Test suite() {
suite.addTestSuite(OracleIncrementalImportTest.class);
suite.addTestSuite(OracleSplitterTest.class);
suite.addTestSuite(OraOopDataDrivenDBInputFormatConnectionCloseTest.class);
suite.addTestSuite(OracleColumnEscapeImportTest.class);
// SQL Server
suite.addTestSuite(SQLServerDatatypeExportDelimitedFileManualTest.class);

5
src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java
View File

@ -38,7 +38,10 @@ public final class MySQLTestUtils {
"sqoop.test.mysql.connectstring.host_url",
"jdbc:mysql://localhost/");
public static final String MYSQL_DATABASE_NAME = "sqooptestdb";
public static final String USER_NAME = System.getProperty("sqoop.test.mysql.username", "sqoop");
public static final String USER_PASS = System.getProperty("sqoop.test.mysql.password", "sqoop");
public static final String MYSQL_DATABASE_NAME = System.getProperty("sqoop.test.mysql.databasename", "sqooptestdb");
public static final String TABLE_NAME = "EMPLOYEES_MYSQL";
public static final String CONNECT_STRING = HOST_URL + MYSQL_DATABASE_NAME;

11
src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
View File

@ -295,8 +295,7 @@ protected String getColName(int i) {
*/
protected void dropTableIfExists(String table) throws SQLException {
Connection conn = getManager().getConnection();
PreparedStatement statement = conn.prepareStatement(
"DROP TABLE " + manager.escapeTableName(table) + " IF EXISTS",
PreparedStatement statement = conn.prepareStatement(dropTableIfExistsCommand(table),
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
try {
statement.executeUpdate();
@ -306,6 +305,10 @@ protected void dropTableIfExists(String table) throws SQLException {
}
}
protected String dropTableIfExistsCommand(String table) {
return "DROP TABLE " + manager.escapeTableName(table) + " IF EXISTS";
}
/**
* Create a table with a set of columns with their names and add a row of values.
* @param colNames Column names
@ -331,7 +334,7 @@ protected void createTableWithColTypesAndNames(String[] colNames,
conn = getManager().getConnection();
for (int i = 0; i < colTypes.length; i++) {
columnDefStr += '"' + colNames[i].toUpperCase() + '"' + " " + colTypes[i];
columnDefStr += manager.escapeColName(colNames[i].toUpperCase()) + " " + colTypes[i];
if (i < colTypes.length - 1) {
columnDefStr += ", ";
}
@ -363,7 +366,7 @@ protected void createTableWithColTypesAndNames(String[] colNames,
String columnListStr = "";
String valueListStr = "";
for (int i = 0; i < colTypes.length; i++) {
columnListStr += '"' + colNames[i].toUpperCase() + '"';
columnListStr += manager.escapeColName(colNames[i].toUpperCase());
valueListStr += vals[count * colTypes.length + i];
if (i < colTypes.length - 1) {
columnListStr += ", ";

112
src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java Normal file
View File

@ -0,0 +1,112 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.manager.mysql;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.manager.MySQLTestUtils;
import com.cloudera.sqoop.testutil.CommonArgs;
import com.cloudera.sqoop.testutil.ImportJobTestCase;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
public class MySqlColumnEscapeImportTest extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog(
MySqlColumnEscapeImportTest.class.getName());
@Override
protected boolean useHsqldbTestServer() {
return false;
}
@Override
protected String getConnectString() {
return MySQLTestUtils.CONNECT_STRING;
}
@Override
protected SqoopOptions getSqoopOptions(Configuration conf) {
SqoopOptions opts = new SqoopOptions(conf);
opts.setUsername(MySQLTestUtils.USER_NAME);
opts.setPassword(MySQLTestUtils.USER_PASS);
return opts;
}
@Override
protected String dropTableIfExistsCommand(String table) {
return "DROP TABLE IF EXISTS " + getManager().escapeTableName(table);
}
@Override
public void tearDown() {
try {
dropTableIfExists(getTableName());
} catch (SQLException e) {
LOG.error("Could not delete test table", e);
}
super.tearDown();
}
protected String [] getArgv() {
ArrayList<String> args = new ArrayList<String>();
CommonArgs.addHadoopFlags(args);
args.add("--connect");
args.add(getConnectString());
args.add("--username");
args.add(MySQLTestUtils.USER_NAME);
args.add("--password");
args.add(MySQLTestUtils.USER_PASS);
args.add("--target-dir");
args.add(getWarehouseDir());
args.add("--num-mappers");
args.add("1");
args.add("--table");
args.add(getTableName());
return args.toArray(new String[0]);
}
public void testEscapeColumnWithDoubleQuote() throws IOException {
String[] colNames = { "column\"withdoublequote" };
String[] types = { "VARCHAR(50)"};
String[] vals = { "'hello, world'"};
createTableWithColTypesAndNames(colNames, types, vals);
String[] args = getArgv();
runImport(args);
Path warehousePath = new Path(this.getWarehouseDir());
Path filePath = new Path(warehousePath, "part-m-00000");
String output = Files.toString(new File(filePath.toString()), Charsets.UTF_8);
assertEquals("hello, world", output.trim());
}
}

110
src/test/org/apache/sqoop/manager/oracle/OracleColumnEscapeImportTest.java Normal file
View File

@ -0,0 +1,110 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.manager.oracle;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.manager.OracleUtils;
import com.cloudera.sqoop.testutil.CommonArgs;
import com.cloudera.sqoop.testutil.ImportJobTestCase;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
public class OracleColumnEscapeImportTest extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog(
OracleColumnEscapeImportTest.class.getName());
@Override
protected boolean useHsqldbTestServer() {
return false;
}
@Override
protected String getConnectString() {
return OracleUtils.CONNECT_STRING;
}
@Override
protected SqoopOptions getSqoopOptions(Configuration conf) {
SqoopOptions opts = new SqoopOptions(conf);
OracleUtils.setOracleAuth(opts);
return opts;
}
@Override
protected void dropTableIfExists(String table) throws SQLException {
OracleUtils.dropTable(table, getManager());
}
@Override
public void tearDown() {
try {
OracleUtils.dropTable(getTableName(), getManager());
} catch (SQLException e) {
LOG.error("Test table could not be dropped", e);
}
super.tearDown();
}
protected String [] getArgv() {
ArrayList<String> args = new ArrayList<String>();
CommonArgs.addHadoopFlags(args);
args.add("--connect");
args.add(getConnectString());
args.add("--username");
args.add(OracleUtils.ORACLE_USER_NAME);
args.add("--password");
args.add(OracleUtils.ORACLE_USER_PASS);
args.add("--target-dir");
args.add(getWarehouseDir());
args.add("--num-mappers");
args.add("1");
args.add("--query");
args.add("select REGEXP_REPLACE(TRIM(" + getColName(0) + "), '\\:','!') from " + getTableName() + " WHERE $CONDITIONS");
return args.toArray(new String[0]);
}
public void testRegexpReplaceEscapeWithSpecialCharacters() throws IOException {
String [] types = { "VARCHAR(50)"};
String [] vals = { "'hello, world:'"};
createTableWithColTypes(types, vals);
String[] args = getArgv();
runImport(args);
Path warehousePath = new Path(this.getWarehouseDir());
Path filePath = new Path(warehousePath, "part-m-00000");
String output = Files.toString(new File(filePath.toString()), Charsets.UTF_8);
assertEquals("hello, world!", output.trim());
}
}