github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-17 01:11:07 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-467: Add support for Oracle BINARY_FLOAT and BINARY_DOUBLE data types

Browse Source 
(Cheolsoo Park via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2012-11-02 12:16:17 -07:00
parent 0a30ae174c
commit 5eb987a785
3 changed files with 104 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/java/org/apache/sqoop/manager/MySQLManager.java
View File

@ -275,8 +275,8 @@ protected String getSchemaQuery() {
private Map<String, String> colTypeNames;
private static final int YEAR_TYPE_OVERWRITE = Types.SMALLINT;
@Override
public String toJavaType(String tableName, String columnName, int sqlType) {
private int overrideSqlType(String tableName, String columnName,
int sqlType) {
if (colTypeNames == null) {
colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
}
@ -284,33 +284,24 @@ public String toJavaType(String tableName, String columnName, int sqlType) {
if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
sqlType = YEAR_TYPE_OVERWRITE;
}
return sqlType;
}
@Override
public String toJavaType(String tableName, String columnName, int sqlType) {
sqlType = overrideSqlType(tableName, columnName, sqlType);
return super.toJavaType(tableName, columnName, sqlType);
}
@Override
public String toHiveType(String tableName, String columnName, int sqlType) {
if (colTypeNames == null) {
colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
}
if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
sqlType = YEAR_TYPE_OVERWRITE;
}
sqlType = overrideSqlType(tableName, columnName, sqlType);
return super.toHiveType(tableName, columnName, sqlType);
}
@Override
public Type toAvroType(String tableName, String columnName, int sqlType) {
if (colTypeNames == null) {
colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
}
if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
sqlType = YEAR_TYPE_OVERWRITE;
}
sqlType = overrideSqlType(tableName, columnName, sqlType);
return super.toAvroType(tableName, columnName, sqlType);
}
}

157
src/java/org/apache/sqoop/manager/OracleManager.java
View File

@ -469,121 +469,90 @@ public ResultSet readTable(String tableName, String[] columns)
return execute(sqlCmd);
}
private Map<String, String> columnTypeNames;
/**
* Resolve a database-specific type to the Java type that should contain it.
* @param tableName table name
* @param columnName column name
* @param sqlType sql data type
* @param colName column name
* @return the name of a Java type to hold the sql datatype, or null if none.
*/
@Override
public String toJavaType(String tableName, String columnName, int sqlType) {
String defaultJavaType = super.toJavaType(tableName, columnName, sqlType);
return (defaultJavaType == null) ? dbToJavaType(sqlType) : defaultJavaType;
private String toDbSpecificJavaType(String tableName, String colName) {
if (columnTypeNames == null) {
columnTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
}
String colTypeName = columnTypeNames.get(colName);
if (colTypeName != null) {
if (colTypeName.equalsIgnoreCase("BINARY_FLOAT")) {
return "Float";
}
if (colTypeName.equalsIgnoreCase("BINARY_DOUBLE")) {
return "Double";
}
if (colTypeName.toUpperCase().startsWith("TIMESTAMP")) {
return "java.sql.Timestamp";
}
}
return null;
}
/**
* Attempt to map sql type to java type.
* Resolve a database-specific type to the Hive type that should contain it.
* @param tableName table name
* @param colName column name
* @return the name of a Hive type to hold the sql datatype, or null if none.
*/
private String toDbSpecificHiveType(String tableName, String colName) {
if (columnTypeNames == null) {
columnTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
}
String colTypeName = columnTypeNames.get(colName);
if (colTypeName != null) {
if (colTypeName.equalsIgnoreCase("BINARY_FLOAT")) {
return "FLOAT";
}
if (colTypeName.equalsIgnoreCase("BINARY_DOUBLE")) {
return "DOUBLE";
}
if (colTypeName.toUpperCase().startsWith("TIMESTAMP")) {
return "STRING";
}
}
return null;
}
/**
* Return java type for SQL type.
* @param tableName table name
* @param columnName column name
* @param sqlType sql type
* @return java type
*/
private String dbToJavaType(int sqlType) {
// load class oracle.jdbc.OracleTypes
// need to use reflection because oracle specific libraries
// are not accessible in this context
Class typeClass = getTypeClass("oracle.jdbc.OracleTypes");
// check if it is TIMESTAMPTZ
int dbType = getDatabaseType(typeClass, "TIMESTAMPTZ");
if (sqlType == dbType) {
return "java.sql.Timestamp";
@Override
public String toJavaType(String tableName, String columnName, int sqlType) {
String javaType = super.toJavaType(tableName, columnName, sqlType);
if (javaType == null) {
javaType = toDbSpecificJavaType(tableName, columnName);
}
// check if it is TIMESTAMPLTZ
dbType = getDatabaseType(typeClass, "TIMESTAMPLTZ");
if (sqlType == dbType) {
return "java.sql.Timestamp";
}
// return null if no java type was found for sqlType
return null;
return javaType;
}
/**
* Attempt to map sql type to hive type.
* Return hive type for SQL type.
* @param tableName table name
* @param columnName column name
* @param sqlType sql data type
* @return hive data type
*/
public String toHiveType(String tableName, String columnName, int sqlType) {
String defaultHiveType = super.toHiveType(tableName, columnName, sqlType);
return (defaultHiveType == null) ? dbToHiveType(sqlType) : defaultHiveType;
}
/**
* Resolve a database-specific type to Hive type.
* @param sqlType sql type
* @return hive type
*/
private String dbToHiveType(int sqlType) {
// load class oracle.jdbc.OracleTypes
// need to use reflection because oracle specific libraries
// are not accessible in this context
Class typeClass = getTypeClass("oracle.jdbc.OracleTypes");
// check if it is TIMESTAMPTZ
int dbType = getDatabaseType(typeClass, "TIMESTAMPTZ");
if (sqlType == dbType) {
return "STRING";
@Override
public String toHiveType(String tableName, String columnName, int sqlType) {
String hiveType = super.toHiveType(tableName, columnName, sqlType);
if (hiveType == null) {
hiveType = toDbSpecificHiveType(tableName, columnName);
}
// check if it is TIMESTAMPLTZ
dbType = getDatabaseType(typeClass, "TIMESTAMPLTZ");
if (sqlType == dbType) {
return "STRING";
}
// return null if no hive type was found for sqlType
return null;
}
/**
* Get database type.
* @param clazz oracle class representing sql types
* @param fieldName field name
* @return value of database type constant
*/
private int getDatabaseType(Class clazz, String fieldName) {
// Need to use reflection to extract constant values because the database
// specific java libraries are not accessible in this context.
int value = -1;
try {
java.lang.reflect.Field field = clazz.getDeclaredField(fieldName);
value = field.getInt(null);
} catch (NoSuchFieldException ex) {
LOG.error("Could not retrieve value for field " + fieldName, ex);
} catch (IllegalAccessException ex) {
LOG.error("Could not retrieve value for field " + fieldName, ex);
}
return value;
}
/**
* Load class by name.
* @param className class name
* @return class instance
*/
private Class getTypeClass(String className) {
// Need to use reflection to load class because the database specific java
// libraries are not accessible in this context.
Class typeClass = null;
try {
typeClass = Class.forName(className);
} catch (ClassNotFoundException ex) {
LOG.error("Could not load class " + className, ex);
}
return typeClass;
return hiveType;
}
@Override

32
src/test/com/cloudera/sqoop/manager/OracleCompatTest.java
View File

@ -184,6 +184,14 @@ protected String getBlobInsertStr(String blobData) {
return sb.toString();
}
protected String getBinaryFloatInsertStr(float f) {
return "TO_BINARY_FLOAT('" + f + "')";
}
protected String getBinaryDoubleInsertStr(double d) {
return "TO_BINARY_DOUBLE('" + d + "')";
}
// Disable this test since Oracle isn't ANSI compliant.
@Override
public void testEmptyStringCol() {
@ -221,5 +229,29 @@ public void testDate2() {
public void testRawVal() {
verifyType("RAW(8)", "'12ABCD'", getVarBinarySeqOutput("12ABCD"), true);
}
public void testBinaryFloat() {
verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(25f), "25.0");
verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(+6.34f), "6.34");
// Max and min are from Oracle DB SQL reference for 10g release 2
float max = 3.40282E+38F;
verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(max), "3.40282E38");
float min = 1.17549E-38F;
verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(min), "1.17549E-38");
}
public void testBinaryDouble() {
verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(0.5d), "0.5");
verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(-1d), "-1.0");
// Max and min are from Oracle DB SQL reference for 10g release 2
double max = 1.79769313486231E+308;
verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(max),
"1.79769313486231E308");
double min = 2.22507485850720E-308;
verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(min),
"2.2250748585072E-308");
}
}