diff --git a/src/java/org/apache/sqoop/manager/MySQLManager.java b/src/java/org/apache/sqoop/manager/MySQLManager.java index a3f586ac..b79c9266 100644 --- a/src/java/org/apache/sqoop/manager/MySQLManager.java +++ b/src/java/org/apache/sqoop/manager/MySQLManager.java @@ -275,8 +275,8 @@ protected String getSchemaQuery() { private Map colTypeNames; private static final int YEAR_TYPE_OVERWRITE = Types.SMALLINT; - @Override - public String toJavaType(String tableName, String columnName, int sqlType) { + private int overrideSqlType(String tableName, String columnName, + int sqlType) { if (colTypeNames == null) { colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery()); } @@ -284,33 +284,24 @@ public String toJavaType(String tableName, String columnName, int sqlType) { if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) { sqlType = YEAR_TYPE_OVERWRITE; } + return sqlType; + } + @Override + public String toJavaType(String tableName, String columnName, int sqlType) { + sqlType = overrideSqlType(tableName, columnName, sqlType); return super.toJavaType(tableName, columnName, sqlType); } @Override public String toHiveType(String tableName, String columnName, int sqlType) { - if (colTypeNames == null) { - colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery()); - } - - if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) { - sqlType = YEAR_TYPE_OVERWRITE; - } - + sqlType = overrideSqlType(tableName, columnName, sqlType); return super.toHiveType(tableName, columnName, sqlType); } @Override public Type toAvroType(String tableName, String columnName, int sqlType) { - if (colTypeNames == null) { - colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery()); - } - - if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) { - sqlType = YEAR_TYPE_OVERWRITE; - } - + sqlType = overrideSqlType(tableName, columnName, sqlType); return super.toAvroType(tableName, columnName, sqlType); } } diff --git a/src/java/org/apache/sqoop/manager/OracleManager.java b/src/java/org/apache/sqoop/manager/OracleManager.java index 6974e1b6..22a057a1 100644 --- a/src/java/org/apache/sqoop/manager/OracleManager.java +++ b/src/java/org/apache/sqoop/manager/OracleManager.java @@ -469,121 +469,90 @@ public ResultSet readTable(String tableName, String[] columns) return execute(sqlCmd); } + private Map columnTypeNames; + /** * Resolve a database-specific type to the Java type that should contain it. * @param tableName table name - * @param columnName column name - * @param sqlType sql data type + * @param colName column name * @return the name of a Java type to hold the sql datatype, or null if none. */ - @Override - public String toJavaType(String tableName, String columnName, int sqlType) { - String defaultJavaType = super.toJavaType(tableName, columnName, sqlType); - return (defaultJavaType == null) ? dbToJavaType(sqlType) : defaultJavaType; + private String toDbSpecificJavaType(String tableName, String colName) { + if (columnTypeNames == null) { + columnTypeNames = getColumnTypeNames(tableName, options.getSqlQuery()); + } + + String colTypeName = columnTypeNames.get(colName); + if (colTypeName != null) { + if (colTypeName.equalsIgnoreCase("BINARY_FLOAT")) { + return "Float"; + } + if (colTypeName.equalsIgnoreCase("BINARY_DOUBLE")) { + return "Double"; + } + if (colTypeName.toUpperCase().startsWith("TIMESTAMP")) { + return "java.sql.Timestamp"; + } + } + return null; } /** - * Attempt to map sql type to java type. + * Resolve a database-specific type to the Hive type that should contain it. + * @param tableName table name + * @param colName column name + * @return the name of a Hive type to hold the sql datatype, or null if none. + */ + private String toDbSpecificHiveType(String tableName, String colName) { + if (columnTypeNames == null) { + columnTypeNames = getColumnTypeNames(tableName, options.getSqlQuery()); + } + + String colTypeName = columnTypeNames.get(colName); + if (colTypeName != null) { + if (colTypeName.equalsIgnoreCase("BINARY_FLOAT")) { + return "FLOAT"; + } + if (colTypeName.equalsIgnoreCase("BINARY_DOUBLE")) { + return "DOUBLE"; + } + if (colTypeName.toUpperCase().startsWith("TIMESTAMP")) { + return "STRING"; + } + } + return null; + } + + /** + * Return java type for SQL type. + * @param tableName table name + * @param columnName column name * @param sqlType sql type * @return java type */ - private String dbToJavaType(int sqlType) { - // load class oracle.jdbc.OracleTypes - // need to use reflection because oracle specific libraries - // are not accessible in this context - Class typeClass = getTypeClass("oracle.jdbc.OracleTypes"); - - // check if it is TIMESTAMPTZ - int dbType = getDatabaseType(typeClass, "TIMESTAMPTZ"); - if (sqlType == dbType) { - return "java.sql.Timestamp"; + @Override + public String toJavaType(String tableName, String columnName, int sqlType) { + String javaType = super.toJavaType(tableName, columnName, sqlType); + if (javaType == null) { + javaType = toDbSpecificJavaType(tableName, columnName); } - - // check if it is TIMESTAMPLTZ - dbType = getDatabaseType(typeClass, "TIMESTAMPLTZ"); - if (sqlType == dbType) { - return "java.sql.Timestamp"; - } - - // return null if no java type was found for sqlType - return null; + return javaType; } /** - * Attempt to map sql type to hive type. + * Return hive type for SQL type. * @param tableName table name * @param columnName column name * @param sqlType sql data type - * @return hive data type - */ - public String toHiveType(String tableName, String columnName, int sqlType) { - String defaultHiveType = super.toHiveType(tableName, columnName, sqlType); - return (defaultHiveType == null) ? dbToHiveType(sqlType) : defaultHiveType; - } - - /** - * Resolve a database-specific type to Hive type. - * @param sqlType sql type * @return hive type */ - private String dbToHiveType(int sqlType) { - // load class oracle.jdbc.OracleTypes - // need to use reflection because oracle specific libraries - // are not accessible in this context - Class typeClass = getTypeClass("oracle.jdbc.OracleTypes"); - - // check if it is TIMESTAMPTZ - int dbType = getDatabaseType(typeClass, "TIMESTAMPTZ"); - if (sqlType == dbType) { - return "STRING"; + @Override + public String toHiveType(String tableName, String columnName, int sqlType) { + String hiveType = super.toHiveType(tableName, columnName, sqlType); + if (hiveType == null) { + hiveType = toDbSpecificHiveType(tableName, columnName); } - - // check if it is TIMESTAMPLTZ - dbType = getDatabaseType(typeClass, "TIMESTAMPLTZ"); - if (sqlType == dbType) { - return "STRING"; - } - - // return null if no hive type was found for sqlType - return null; - } - - /** - * Get database type. - * @param clazz oracle class representing sql types - * @param fieldName field name - * @return value of database type constant - */ - private int getDatabaseType(Class clazz, String fieldName) { - // Need to use reflection to extract constant values because the database - // specific java libraries are not accessible in this context. - int value = -1; - try { - java.lang.reflect.Field field = clazz.getDeclaredField(fieldName); - value = field.getInt(null); - } catch (NoSuchFieldException ex) { - LOG.error("Could not retrieve value for field " + fieldName, ex); - } catch (IllegalAccessException ex) { - LOG.error("Could not retrieve value for field " + fieldName, ex); - } - return value; - } - - /** - * Load class by name. - * @param className class name - * @return class instance - */ - private Class getTypeClass(String className) { - // Need to use reflection to load class because the database specific java - // libraries are not accessible in this context. - Class typeClass = null; - try { - typeClass = Class.forName(className); - } catch (ClassNotFoundException ex) { - LOG.error("Could not load class " + className, ex); - } - return typeClass; + return hiveType; } @Override diff --git a/src/test/com/cloudera/sqoop/manager/OracleCompatTest.java b/src/test/com/cloudera/sqoop/manager/OracleCompatTest.java index 309a960d..5bf920a3 100644 --- a/src/test/com/cloudera/sqoop/manager/OracleCompatTest.java +++ b/src/test/com/cloudera/sqoop/manager/OracleCompatTest.java @@ -184,6 +184,14 @@ protected String getBlobInsertStr(String blobData) { return sb.toString(); } + protected String getBinaryFloatInsertStr(float f) { + return "TO_BINARY_FLOAT('" + f + "')"; + } + + protected String getBinaryDoubleInsertStr(double d) { + return "TO_BINARY_DOUBLE('" + d + "')"; + } + // Disable this test since Oracle isn't ANSI compliant. @Override public void testEmptyStringCol() { @@ -221,5 +229,29 @@ public void testDate2() { public void testRawVal() { verifyType("RAW(8)", "'12ABCD'", getVarBinarySeqOutput("12ABCD"), true); } + + public void testBinaryFloat() { + verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(25f), "25.0"); + verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(+6.34f), "6.34"); + + // Max and min are from Oracle DB SQL reference for 10g release 2 + float max = 3.40282E+38F; + verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(max), "3.40282E38"); + float min = 1.17549E-38F; + verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(min), "1.17549E-38"); + } + + public void testBinaryDouble() { + verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(0.5d), "0.5"); + verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(-1d), "-1.0"); + + // Max and min are from Oracle DB SQL reference for 10g release 2 + double max = 1.79769313486231E+308; + verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(max), + "1.79769313486231E308"); + double min = 2.22507485850720E-308; + verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(min), + "2.2250748585072E-308"); + } }