diff --git a/src/java/org/apache/sqoop/avro/AvroUtil.java b/src/java/org/apache/sqoop/avro/AvroUtil.java index 319be0f7..ee29f140 100644 --- a/src/java/org/apache/sqoop/avro/AvroUtil.java +++ b/src/java/org/apache/sqoop/avro/AvroUtil.java @@ -114,11 +114,25 @@ public static String toAvroColumn(String column) { * Format candidate to avro specifics */ public static String toAvroIdentifier(String candidate) { - String formattedCandidate = candidate.replaceAll("\\W+", "_"); - if (formattedCandidate.substring(0,1).matches("[a-zA-Z_]")) { - return formattedCandidate; + char[] data = candidate.toCharArray(); + boolean skip = false; + int stringIndex = 0; + + for (char c:data) { + if (Character.isLetterOrDigit(c) || c == '_') { + data[stringIndex++] = c; + skip = false; + } else if(!skip) { + data[stringIndex++] = '_'; + skip = true; + } + } + + char initial = data[0]; + if (Character.isLetter(initial) || initial == '_') { + return new String(data, 0, stringIndex); } else { - return "AVRO_" + formattedCandidate; + return "AVRO_".concat(new String(data, 0, stringIndex)); } } diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java index 23a9c418..9d91887d 100644 --- a/src/java/org/apache/sqoop/orm/ClassWriter.java +++ b/src/java/org/apache/sqoop/orm/ClassWriter.java @@ -1064,57 +1064,49 @@ private void myGenerateCloneMethod(Map columnTypes, * @param colNames - ordered list of column names for table. * @param sb - StringBuilder to append code to */ - private void generateSetField(Map columnTypes, - String [] colNames, String [] rawColNames, StringBuilder sb) { - - int numberOfMethods = - this.getNumberOfMethods(colNames, maxColumnsPerMethod); - - sb.append(" public void setField(String __fieldName, Object __fieldVal) " - + "{\n"); - if (numberOfMethods > 1) { - boolean first = true; - for (int i = 0; i < numberOfMethods; ++i) { - if (!first) { - sb.append(" else"); - } - sb.append(" if (this.setField" + i - + "(__fieldName, __fieldVal)) {\n"); - sb.append(" return;\n"); - sb.append(" }\n"); - first = false; - } - } else { - boolean first = true; - for (int i = 0; i < colNames.length; i++) { - int sqlType = columnTypes.get(colNames[i]); - String javaType = toJavaType(colNames[i], sqlType); - if (null == javaType) { - continue; - } else { - if (!first) { - sb.append(" else"); - } - - sb.append(" if (\"" + serializeRawColName(rawColNames[i]) + "\".equals(__fieldName)) {\n"); - sb.append(" this." + colNames[i] + " = (" + javaType - + ") __fieldVal;\n"); - sb.append(" }\n"); - first = false; - } - } - } - sb.append(" else {\n"); - sb.append(" throw new RuntimeException("); - sb.append("\"No such field: \" + __fieldName);\n"); - sb.append(" }\n"); - sb.append(" }\n"); - - for (int i = 0; i < numberOfMethods; ++i) { - myGenerateSetField(columnTypes, colNames, rawColNames, sb, i, maxColumnsPerMethod); - } + private void generateSetField(Map columnTypes, String[] colNames, String[] rawColNames, + StringBuilder sb) { + String sep = System.getProperty("line.separator"); + sb.append(" public void setField(String __fieldName, Object __fieldVal) " + "{" + sep); + sb.append(" if (!setters.containsKey(__fieldName)) {" + sep); + sb.append(" throw new RuntimeException(\"No such field:\"+__fieldName);" + sep); + sb.append(" }" + sep); + sb.append(" setters.get(__fieldName).setField(__fieldVal);" + sep); + sb.append(" }\n" + sep); } + private void generateConstructorAndInitMethods(Map colTypes, String[] colNames, String[] rawColNames, + String typeName, StringBuilder sb) { + String sep = System.getProperty("line.separator"); + int numberOfMethods = getNumberOfMethods(colNames, maxColumnsPerMethod); + for (int methodNumber = 0; methodNumber < numberOfMethods; ++methodNumber) { + sb.append(" private void init" + methodNumber + "() {" + sep); + for (int i = methodNumber * maxColumnsPerMethod; i < topBoundary(colNames, methodNumber, + maxColumnsPerMethod); ++i) { + String colName = colNames[i]; + String rawColName = rawColNames[i]; + int sqlType = colTypes.get(colName); + String javaType = toJavaType(colName, sqlType); + if (null == javaType) { + LOG.error("Cannot resolve SQL type " + sqlType); + continue; + } else { + sb.append(" setters.put(\"" + serializeRawColName(rawColName) + "\", new FieldSetterCommand() {" + sep); + sb.append(" @Override" + sep); + sb.append(" public void setField(Object value) {" + sep); + sb.append(" " + colName + " = (" + javaType + ")value;" + sep); + sb.append(" }" + sep); + sb.append(" });" + sep); + } + } + sb.append(" }" + sep); + } + sb.append(" public " + typeName + "() {" + sep); + for (int i = 0; i < numberOfMethods; ++i) { + sb.append(" init" + i + "();" + sep); + } + sb.append(" }" + sep); + } /** * Raw column name is a column name as it was created on database and we need to serialize it between * double quotes into java class that will be further complied with javac. Various databases supports @@ -1184,7 +1176,7 @@ private void generateGetFieldMap(Map columnTypes, sb.append(" public Map getFieldMap() {\n"); sb.append(" Map __sqoop$field_map = " - + "new TreeMap();\n"); + + "new HashMap();\n"); if (numberOfMethods > 1) { for (int i = 0; i < numberOfMethods; ++i) { sb.append(" this.getFieldMap" + i + "(__sqoop$field_map);\n"); @@ -1934,7 +1926,7 @@ private StringBuilder generateClassForColumns( sb.append("import java.util.Iterator;\n"); sb.append("import java.util.List;\n"); sb.append("import java.util.Map;\n"); - sb.append("import java.util.TreeMap;\n"); + sb.append("import java.util.HashMap;\n"); sb.append("\n"); String className = tableNameInfo.getShortClassForTable(tableName); @@ -1944,7 +1936,12 @@ private StringBuilder generateClassForColumns( + CLASS_WRITER_VERSION + ";\n"); sb.append( " public int getClassFormatVersion() { return PROTOCOL_VERSION; }\n"); + sb.append(" public static interface FieldSetterCommand {"); + sb.append(" void setField(Object value);"); + sb.append(" }"); sb.append(" protected ResultSet __cur_result_set;\n"); + sb.append(" private Map setters = new HashMap();\n"); + generateConstructorAndInitMethods(columnTypes, colNames, rawColNames, className, sb); generateFields(columnTypes, colNames, className, sb); generateEquals(columnTypes, colNames, className, sb); generateDbRead(columnTypes, colNames, sb); diff --git a/src/java/org/apache/sqoop/orm/CompilationManager.java b/src/java/org/apache/sqoop/orm/CompilationManager.java index ce165e8e..0a2a87f2 100644 --- a/src/java/org/apache/sqoop/orm/CompilationManager.java +++ b/src/java/org/apache/sqoop/orm/CompilationManager.java @@ -296,11 +296,15 @@ private void addClassFilesFromDir(File dir, JarOutputStream jstream) // we only record the subdir parts in the zip entry. String fullPath = entry.getAbsolutePath(); String chompedPath = fullPath.substring(baseDirName.length()); + int indexOfDollarSign = chompedPath.indexOf("$"); + String innerTypesChompedPath = chompedPath + .substring(0, indexOfDollarSign == -1 ? chompedPath.length() : indexOfDollarSign); boolean include = chompedPath.endsWith(".class") - && sources.contains( + && (sources.contains( chompedPath.substring(0, chompedPath.length() - ".class".length()) - + ".java"); + + ".java") + || sources.contains(innerTypesChompedPath + ".java")); if (include) { // include this file. diff --git a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java index 498db730..10a09696 100644 --- a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java +++ b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java @@ -26,6 +26,7 @@ import java.sql.Connection; import java.sql.Statement; import java.sql.SQLException; +import java.util.Random; import java.util.jar.JarEntry; import java.util.jar.JarInputStream; @@ -57,6 +58,9 @@ public class TestClassWriter extends TestCase { public static final Log LOG = LogFactory.getLog(TestClassWriter.class.getName()); + private static final String WIDE_TABLE_NAME = "WIDETABLE"; + private static final int WIDE_TABLE_COLUMN_COUNT = 800; + private static final int WIDE_TABLE_ROW_COUNT = 20_000; // instance variables populated during setUp, used during tests private HsqldbTestServer testServer; @@ -122,12 +126,16 @@ public void tearDown() { static final String JAR_GEN_DIR = ImportJobTestCase.TEMP_BASE_DIR + "sqoop/test/jargen"; + private File runGenerationTest(String[] argv, String classNameToCheck) { + return runGenerationTest(argv, classNameToCheck, HsqldbTestServer.getTableName()); + } + /** * Run a test to verify that we can generate code and it emits the output * files where we expect them. * @return */ - private File runGenerationTest(String [] argv, String classNameToCheck) { + private File runGenerationTest(String[] argv, String classNameToCheck, String tableName) { File codeGenDirFile = new File(CODE_GEN_DIR); File classGenDirFile = new File(JAR_GEN_DIR); @@ -140,7 +148,7 @@ private File runGenerationTest(String [] argv, String classNameToCheck) { CompilationManager compileMgr = new CompilationManager(options); ClassWriter writer = new ClassWriter(options, manager, - HsqldbTestServer.getTableName(), compileMgr); + tableName, compileMgr); try { writer.generate(); @@ -675,4 +683,55 @@ public void testNoClassGeneration() throws Exception { }; runFailedGenerationTest(argv, HsqldbTestServer.getTableName()); } + + @Test(timeout = 10000) + public void testWideTableClassGeneration() throws Exception { + createWideTable(); + options = new SqoopOptions(HsqldbTestServer.getDbUrl(), WIDE_TABLE_NAME); + + // Set the option strings in an "argv" to redirect our srcdir and bindir. + String [] argv = { + "--bindir", + JAR_GEN_DIR, + "--outdir", + CODE_GEN_DIR, + }; + + File ormJarFile = runGenerationTest(argv, WIDE_TABLE_NAME, WIDE_TABLE_NAME); + + ClassLoader prevClassLoader = ClassLoaderStack.addJarFile(ormJarFile.getCanonicalPath(), + WIDE_TABLE_NAME); + Class tableClass = Class.forName(WIDE_TABLE_NAME, true, + Thread.currentThread().getContextClassLoader()); + + Object instance = tableClass.newInstance(); + Method setterMethod = tableClass.getMethod("setField", String.class, Object.class); + Random random = new Random(0); + for (int j = 0; j < WIDE_TABLE_ROW_COUNT; ++j) { + for (int i = 0; i < WIDE_TABLE_COLUMN_COUNT; ++i) { + setterMethod.invoke(instance, "INTFIELD" + i, random.nextInt()); + } + } + + if (null != prevClassLoader) { + ClassLoaderStack.setCurrentClassLoader(prevClassLoader); + } + } + + private void createWideTable() throws Exception { + try (Connection conn = testServer.getConnection(); Statement stmt = conn.createStatement();) { + stmt.executeUpdate("DROP TABLE \"" + WIDE_TABLE_NAME + "\" IF EXISTS"); + StringBuilder sb = new StringBuilder("CREATE TABLE \"" + WIDE_TABLE_NAME + "\" ("); + for (int i = 0; i < WIDE_TABLE_COLUMN_COUNT; ++i) { + sb.append("intField" + i + " INT"); + if (i < WIDE_TABLE_COLUMN_COUNT - 1) { + sb.append(","); + } else { + sb.append(")"); + } + } + stmt.executeUpdate(sb.toString()); + conn.commit(); + } + } }