github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 18:49:55 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3038: export to table with "value" column causes all null values in the "value" column

Browse Source 
(Anna Szonyi via Attila Szabo)
This commit is contained in:
Attila Szabo 2016-11-08 19:10:00 +01:00
parent c3e868ce90
commit 64fd3d2d32
3 changed files with 38 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/java/org/apache/sqoop/orm/ClassWriter.java
View File

@ -1099,7 +1099,7 @@ private void generateConstructorAndInitMethods(Map<String, Integer> colTypes, St
sb.append(" setters.put(\"" + serializeRawColName(rawColName) + "\", new FieldSetterCommand() {" + sep); sb.append(" setters.put(\"" + serializeRawColName(rawColName) + "\", new FieldSetterCommand() {" + sep);
sb.append(" @Override" + sep); sb.append(" @Override" + sep);
sb.append(" public void setField(Object value) {" + sep); sb.append(" public void setField(Object value) {" + sep);
sb.append(" " + colName + " = (" + javaType + ")value;" + sep); sb.append(" " +typeName+".this." + colName + " = (" + javaType + ")value;" + sep);
sb.append(" }" + sep); sb.append(" }" + sep);
sb.append(" });" + sep); sb.append(" });" + sep);
} }

27
src/test/org/apache/sqoop/hcat/HCatalogExportTest.java
View File

@ -102,8 +102,8 @@ protected void assertColMinAndMax(String colName, ColumnGenerator generator)
Object expectedMin = generator.getDBValue(minId); Object expectedMin = generator.getDBValue(minId);
Object expectedMax = generator.getDBValue(maxId); Object expectedMax = generator.getDBValue(maxId);
utils.assertSqlColValForRowId(conn, table, minId, colName, expectedMin); utils.assertSqlColValForRowId(conn, table, minId, colName, true, expectedMin);
utils.assertSqlColValForRowId(conn, table, maxId, colName, expectedMax); utils.assertSqlColValForRowId(conn, table, maxId, colName, true, expectedMax);
} }
protected void runHCatExport(List<String> addlArgsArray, protected void runHCatExport(List<String> addlArgsArray,
@ -111,7 +111,7 @@ protected void runHCatExport(List<String> addlArgsArray,
ColumnGenerator[] cols) throws Exception { ColumnGenerator[] cols) throws Exception {
utils.createHCatTable(CreateMode.CREATE_AND_LOAD, utils.createHCatTable(CreateMode.CREATE_AND_LOAD,
totalRecords, table, cols); totalRecords, table, cols);
utils.createSqlTable(getConnection(), true, totalRecords, table, cols); utils.createSqlTable(getConnection(), true, totalRecords, table, true, cols);
Map<String, String> addlArgsMap = utils.getAddlTestArgs(); Map<String, String> addlArgsMap = utils.getAddlTestArgs();
addlArgsArray.add("--verbose"); addlArgsArray.add("--verbose");
addlArgsArray.add("-m"); addlArgsArray.add("-m");
@ -138,7 +138,7 @@ protected void runHCatExport(List<String> addlArgsArray,
runExport(exportArgs); runExport(exportArgs);
verifyExport(totalRecords); verifyExport(totalRecords);
for (int i = 0; i < cols.length; i++) { for (int i = 0; i < cols.length; i++) {
assertColMinAndMax(HCatalogTestUtils.forIdx(i), cols[i]); assertColMinAndMax(cols[i].getName(), cols[i]);
} }
} }
@ -166,6 +166,23 @@ public void testIntTypes() throws Exception {
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols); runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
} }
public void testExportWithColumnNameValue() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
String valueColumnThatHadIssuesWithClassWriter = "value";
ColumnGenerator[] cols = new ColumnGenerator[] {
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
"boolean", Types.BOOLEAN, HCatFieldSchema.Type.BOOLEAN, 0, 0,
Boolean.TRUE, Boolean.TRUE, KeyType.NOT_A_KEY),
HCatalogTestUtils.colGenerator(valueColumnThatHadIssuesWithClassWriter,
"int", Types.INTEGER, HCatFieldSchema.Type.INT, 5, 5, 10,
10, KeyType.NOT_A_KEY)
};
List<String> addlArgsArray = new ArrayList<String>();
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
public void testFloatTypes() throws Exception { public void testFloatTypes() throws Exception {
final int TOTAL_RECORDS = 1 * 10; final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase(); String table = getTableName().toUpperCase();
@ -303,7 +320,7 @@ public void testColumnProjection() throws Exception {
}; };
List<String> addlArgsArray = new ArrayList<String>(); List<String> addlArgsArray = new ArrayList<String>();
addlArgsArray.add("--columns"); addlArgsArray.add("--columns");
addlArgsArray.add("ID,MSG"); addlArgsArray.add("id,msg");
runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols); runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
} }

23
src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
View File

@ -560,6 +560,7 @@ public static void assertEquals(Object expectedVal,
} }
} }
/** /**
* Verify that on a given row, a column has a given value. * Verify that on a given row, a column has a given value.
* *
@ -567,12 +568,12 @@ public static void assertEquals(Object expectedVal,
* the id column specifying the row to test. * the id column specifying the row to test.
*/ */
public void assertSqlColValForRowId(Connection conn, public void assertSqlColValForRowId(Connection conn,
String table, int id, String colName, String table, int id, String colName, boolean escapeId,
Object expectedVal) throws SQLException { Object expectedVal) throws SQLException {
LOG.info("Verifying column " + colName + " has value " + expectedVal); LOG.info("Verifying column " + colName + " has value " + expectedVal);
String escapeStr = escapeId? "\"" : "";
PreparedStatement statement = conn.prepareStatement( PreparedStatement statement = conn.prepareStatement(
"SELECT " + colName + " FROM " + table + " WHERE id = " + id, "SELECT "+escapeStr + colName + escapeStr+" FROM " + table + " WHERE "+escapeStr+"id"+escapeStr+" = " + id,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
Object actualVal = null; Object actualVal = null;
try { try {
@ -590,6 +591,7 @@ public void assertSqlColValForRowId(Connection conn,
assertEquals(expectedVal, actualVal); assertEquals(expectedVal, actualVal);
} }
/** /**
* Verify that on a given row, a column has a given value. * Verify that on a given row, a column has a given value.
* *
@ -652,12 +654,13 @@ public static String getSqlDropTableStatement(String tableName) {
return "DROP TABLE " + tableName; return "DROP TABLE " + tableName;
} }
public static String getSqlCreateTableStatement(String tableName, public static String getSqlCreateTableStatement(String tableName, boolean escapeIdMsgCol,
ColumnGenerator... extraCols) { ColumnGenerator... extraCols) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append("CREATE TABLE "); sb.append("CREATE TABLE ");
sb.append(tableName); sb.append(tableName);
sb.append(" (ID INT NOT NULL PRIMARY KEY, MSG VARCHAR(64)"); String escapeStr = escapeIdMsgCol? "\"" : "";
sb.append(" ("+escapeStr+"id"+escapeStr+" INT NOT NULL PRIMARY KEY, "+escapeStr+"msg"+escapeStr+" VARCHAR(64)");
int colNum = 0; int colNum = 0;
for (ColumnGenerator gen : extraCols) { for (ColumnGenerator gen : extraCols) {
sb.append(", \"" + gen.getName() + "\" " + gen.getDBTypeString()); sb.append(", \"" + gen.getName() + "\" " + gen.getDBTypeString());
@ -686,9 +689,13 @@ public static String getSqlInsertTableStatement(String tableName,
LOG.debug("Generated SQL insert table command : " + s); LOG.debug("Generated SQL insert table command : " + s);
return s; return s;
} }
public void createSqlTable(Connection conn, boolean generateOnly, public void createSqlTable(Connection conn, boolean generateOnly,
int count, String table, ColumnGenerator... extraCols) int count, String table, ColumnGenerator... extraCols)
throws Exception {
createSqlTable(conn, generateOnly, count, table, false, extraCols);
}
public void createSqlTable(Connection conn, boolean generateOnly,
int count, String table, boolean escapeIdMsgCols, ColumnGenerator... extraCols)
throws Exception { throws Exception {
PreparedStatement statement = conn.prepareStatement( PreparedStatement statement = conn.prepareStatement(
getSqlDropTableStatement(table), getSqlDropTableStatement(table),
@ -702,7 +709,7 @@ public void createSqlTable(Connection conn, boolean generateOnly,
statement.close(); statement.close();
} }
statement = conn.prepareStatement( statement = conn.prepareStatement(
getSqlCreateTableStatement(table, extraCols), getSqlCreateTableStatement(table, escapeIdMsgCols ,extraCols),
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
try { try {
statement.executeUpdate(); statement.executeUpdate();