From 64fd3d2d32b4ef21f72c92db152c4e6fbfe61d15 Mon Sep 17 00:00:00 2001 From: Attila Szabo Date: Tue, 8 Nov 2016 19:10:00 +0100 Subject: [PATCH] SQOOP-3038: export to table with "value" column causes all null values in the "value" column (Anna Szonyi via Attila Szabo) --- .../org/apache/sqoop/orm/ClassWriter.java | 2 +- .../apache/sqoop/hcat/HCatalogExportTest.java | 27 +++++++++++++++---- .../apache/sqoop/hcat/HCatalogTestUtils.java | 23 ++++++++++------ 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java index e2894299..6f6e66b5 100644 --- a/src/java/org/apache/sqoop/orm/ClassWriter.java +++ b/src/java/org/apache/sqoop/orm/ClassWriter.java @@ -1099,7 +1099,7 @@ private void generateConstructorAndInitMethods(Map colTypes, St sb.append(" setters.put(\"" + serializeRawColName(rawColName) + "\", new FieldSetterCommand() {" + sep); sb.append(" @Override" + sep); sb.append(" public void setField(Object value) {" + sep); - sb.append(" " + colName + " = (" + javaType + ")value;" + sep); + sb.append(" " +typeName+".this." + colName + " = (" + javaType + ")value;" + sep); sb.append(" }" + sep); sb.append(" });" + sep); } diff --git a/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java index 6f87a182..ff3dde1c 100644 --- a/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java +++ b/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java @@ -102,8 +102,8 @@ protected void assertColMinAndMax(String colName, ColumnGenerator generator) Object expectedMin = generator.getDBValue(minId); Object expectedMax = generator.getDBValue(maxId); - utils.assertSqlColValForRowId(conn, table, minId, colName, expectedMin); - utils.assertSqlColValForRowId(conn, table, maxId, colName, expectedMax); + utils.assertSqlColValForRowId(conn, table, minId, colName, true, expectedMin); + utils.assertSqlColValForRowId(conn, table, maxId, colName, true, expectedMax); } protected void runHCatExport(List addlArgsArray, @@ -111,7 +111,7 @@ protected void runHCatExport(List addlArgsArray, ColumnGenerator[] cols) throws Exception { utils.createHCatTable(CreateMode.CREATE_AND_LOAD, totalRecords, table, cols); - utils.createSqlTable(getConnection(), true, totalRecords, table, cols); + utils.createSqlTable(getConnection(), true, totalRecords, table, true, cols); Map addlArgsMap = utils.getAddlTestArgs(); addlArgsArray.add("--verbose"); addlArgsArray.add("-m"); @@ -138,7 +138,7 @@ protected void runHCatExport(List addlArgsArray, runExport(exportArgs); verifyExport(totalRecords); for (int i = 0; i < cols.length; i++) { - assertColMinAndMax(HCatalogTestUtils.forIdx(i), cols[i]); + assertColMinAndMax(cols[i].getName(), cols[i]); } } @@ -166,6 +166,23 @@ public void testIntTypes() throws Exception { runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols); } + public void testExportWithColumnNameValue() throws Exception { + final int TOTAL_RECORDS = 1 * 10; + String table = getTableName().toUpperCase(); + String valueColumnThatHadIssuesWithClassWriter = "value"; + ColumnGenerator[] cols = new ColumnGenerator[] { + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0), + "boolean", Types.BOOLEAN, HCatFieldSchema.Type.BOOLEAN, 0, 0, + Boolean.TRUE, Boolean.TRUE, KeyType.NOT_A_KEY), + HCatalogTestUtils.colGenerator(valueColumnThatHadIssuesWithClassWriter, + "int", Types.INTEGER, HCatFieldSchema.Type.INT, 5, 5, 10, + 10, KeyType.NOT_A_KEY) + }; + List addlArgsArray = new ArrayList(); + runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols); + } + + public void testFloatTypes() throws Exception { final int TOTAL_RECORDS = 1 * 10; String table = getTableName().toUpperCase(); @@ -303,7 +320,7 @@ public void testColumnProjection() throws Exception { }; List addlArgsArray = new ArrayList(); addlArgsArray.add("--columns"); - addlArgsArray.add("ID,MSG"); + addlArgsArray.add("id,msg"); runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols); } diff --git a/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java b/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java index f4b1ea9e..fb4a200f 100644 --- a/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java +++ b/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java @@ -560,6 +560,7 @@ public static void assertEquals(Object expectedVal, } } + /** * Verify that on a given row, a column has a given value. * @@ -567,12 +568,12 @@ public static void assertEquals(Object expectedVal, * the id column specifying the row to test. */ public void assertSqlColValForRowId(Connection conn, - String table, int id, String colName, + String table, int id, String colName, boolean escapeId, Object expectedVal) throws SQLException { LOG.info("Verifying column " + colName + " has value " + expectedVal); - + String escapeStr = escapeId? "\"" : ""; PreparedStatement statement = conn.prepareStatement( - "SELECT " + colName + " FROM " + table + " WHERE id = " + id, + "SELECT "+escapeStr + colName + escapeStr+" FROM " + table + " WHERE "+escapeStr+"id"+escapeStr+" = " + id, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); Object actualVal = null; try { @@ -590,6 +591,7 @@ public void assertSqlColValForRowId(Connection conn, assertEquals(expectedVal, actualVal); } + /** * Verify that on a given row, a column has a given value. * @@ -652,12 +654,13 @@ public static String getSqlDropTableStatement(String tableName) { return "DROP TABLE " + tableName; } - public static String getSqlCreateTableStatement(String tableName, + public static String getSqlCreateTableStatement(String tableName, boolean escapeIdMsgCol, ColumnGenerator... extraCols) { StringBuilder sb = new StringBuilder(); sb.append("CREATE TABLE "); sb.append(tableName); - sb.append(" (ID INT NOT NULL PRIMARY KEY, MSG VARCHAR(64)"); + String escapeStr = escapeIdMsgCol? "\"" : ""; + sb.append(" ("+escapeStr+"id"+escapeStr+" INT NOT NULL PRIMARY KEY, "+escapeStr+"msg"+escapeStr+" VARCHAR(64)"); int colNum = 0; for (ColumnGenerator gen : extraCols) { sb.append(", \"" + gen.getName() + "\" " + gen.getDBTypeString()); @@ -686,9 +689,13 @@ public static String getSqlInsertTableStatement(String tableName, LOG.debug("Generated SQL insert table command : " + s); return s; } - public void createSqlTable(Connection conn, boolean generateOnly, - int count, String table, ColumnGenerator... extraCols) + int count, String table, ColumnGenerator... extraCols) + throws Exception { + createSqlTable(conn, generateOnly, count, table, false, extraCols); + } + public void createSqlTable(Connection conn, boolean generateOnly, + int count, String table, boolean escapeIdMsgCols, ColumnGenerator... extraCols) throws Exception { PreparedStatement statement = conn.prepareStatement( getSqlDropTableStatement(table), @@ -702,7 +709,7 @@ public void createSqlTable(Connection conn, boolean generateOnly, statement.close(); } statement = conn.prepareStatement( - getSqlCreateTableStatement(table, extraCols), + getSqlCreateTableStatement(table, escapeIdMsgCols ,extraCols), ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); try { statement.executeUpdate();