github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-04 03:29:26 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3233: SqoopHCatImportHelper.convertNumberTypes check for Varchar instead of Char

Browse Source 
(Ferenc Szabo via Szabolcs Vasas)
This commit is contained in:
Szabolcs Vasas 2017-12-08 11:39:24 +01:00
parent a50869437e
commit 855c56fef8
2 changed files with 56 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java
View File

@ -335,9 +335,7 @@ private Object converDateTypes(Object val, HCatFieldSchema hfs) {
private Object convertStringTypes(Object val, HCatFieldSchema hfs) { private Object convertStringTypes(Object val, HCatFieldSchema hfs) {
HCatFieldSchema.Type hfsType = hfs.getType(); HCatFieldSchema.Type hfsType = hfs.getType();
if (hfsType == HCatFieldSchema.Type.STRING if (isStringType(hfsType)) {
|| hfsType == HCatFieldSchema.Type.VARCHAR
|| hfsType == HCatFieldSchema.Type.CHAR) {
String str = val.toString(); String str = val.toString();
if (doHiveDelimsReplacement) { if (doHiveDelimsReplacement) {
str = FieldFormatter.hiveStringReplaceDelims(str, str = FieldFormatter.hiveStringReplaceDelims(str,
@ -400,21 +398,63 @@ private Object convertNumberTypes(Object val, HCatFieldSchema hfs) {
return null; return null;
} }
if (val instanceof BigDecimal if (val instanceof BigDecimal
&& hfsType == HCatFieldSchema.Type.STRING && isStringType(hfsType)) {
|| hfsType == HCatFieldSchema.Type.VARCHAR
|| hfsType == HCatFieldSchema.Type.CHAR) {
BigDecimal bd = (BigDecimal) val; BigDecimal bd = (BigDecimal) val;
return convertBigDecimalToTextTypes(hfs, hfsType, bd);
}
Number n = (Number) val;
return convertNumberToAnyType(hfs, hfsType, n);
}
private boolean isStringType(HCatFieldSchema.Type hfsType) {
return hfsType == HCatFieldSchema.Type.STRING
|| hfsType == HCatFieldSchema.Type.VARCHAR
|| hfsType == HCatFieldSchema.Type.CHAR;
}
private Object convertNumberToAnyType(HCatFieldSchema hfs, HCatFieldSchema.Type hfsType, Number number) {
if (hfsType == HCatFieldSchema.Type.TINYINT) {
return number.byteValue();
} else if (hfsType == HCatFieldSchema.Type.SMALLINT) {
return number.shortValue();
} else if (hfsType == HCatFieldSchema.Type.INT) {
return number.intValue();
} else if (hfsType == HCatFieldSchema.Type.BIGINT) {
return number.longValue();
} else if (hfsType == HCatFieldSchema.Type.FLOAT) {
return number.floatValue();
} else if (hfsType == HCatFieldSchema.Type.DOUBLE) {
return number.doubleValue();
} else if (hfsType == HCatFieldSchema.Type.BOOLEAN) {
return number.byteValue() == 0 ? Boolean.FALSE : Boolean.TRUE;
} else if (hfsType == HCatFieldSchema.Type.STRING) {
return number.toString();
} else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
HiveVarchar hvc = new HiveVarchar(number.toString(), vti.getLength());
return hvc;
} else if (hfsType == HCatFieldSchema.Type.CHAR) {
CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
HiveChar hChar = new HiveChar(number.toString(), cti.getLength());
return hChar;
} else if (hfsType == HCatFieldSchema.Type.DECIMAL) {
return convertNumberIntoHiveDecimal(number);
}
return null;
}
private Object convertBigDecimalToTextTypes(HCatFieldSchema hfs, HCatFieldSchema.Type hfsType, BigDecimal bigDecimal) {
String bdStr = null; String bdStr = null;
if (bigDecimalFormatString) { if (bigDecimalFormatString) {
bdStr = bd.toPlainString(); bdStr = bigDecimal.toPlainString();
} else { } else {
bdStr = bd.toString(); bdStr = bigDecimal.toString();
} }
if (hfsType == HCatFieldSchema.Type.VARCHAR) { if (hfsType == HCatFieldSchema.Type.VARCHAR) {
VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo(); VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
HiveVarchar hvc = new HiveVarchar(bdStr, vti.getLength()); HiveVarchar hvc = new HiveVarchar(bdStr, vti.getLength());
return hvc; return hvc;
} else if (hfsType == HCatFieldSchema.Type.VARCHAR) { } else if (hfsType == HCatFieldSchema.Type.CHAR) {
CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo(); CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
HiveChar hChar = new HiveChar(bdStr, cti.getLength()); HiveChar hChar = new HiveChar(bdStr, cti.getLength());
return hChar; return hChar;
@ -422,36 +462,6 @@ private Object convertNumberTypes(Object val, HCatFieldSchema hfs) {
return bdStr; return bdStr;
} }
} }
Number n = (Number) val;
if (hfsType == HCatFieldSchema.Type.TINYINT) {
return n.byteValue();
} else if (hfsType == HCatFieldSchema.Type.SMALLINT) {
return n.shortValue();
} else if (hfsType == HCatFieldSchema.Type.INT) {
return n.intValue();
} else if (hfsType == HCatFieldSchema.Type.BIGINT) {
return n.longValue();
} else if (hfsType == HCatFieldSchema.Type.FLOAT) {
return n.floatValue();
} else if (hfsType == HCatFieldSchema.Type.DOUBLE) {
return n.doubleValue();
} else if (hfsType == HCatFieldSchema.Type.BOOLEAN) {
return n.byteValue() == 0 ? Boolean.FALSE : Boolean.TRUE;
} else if (hfsType == HCatFieldSchema.Type.STRING) {
return n.toString();
} else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
HiveVarchar hvc = new HiveVarchar(val.toString(), vti.getLength());
return hvc;
} else if (hfsType == HCatFieldSchema.Type.CHAR) {
CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
HiveChar hChar = new HiveChar(val.toString(), cti.getLength());
return hChar;
} else if (hfsType == HCatFieldSchema.Type.DECIMAL) {
return convertNumberIntoHiveDecimal(n);
}
return null;
}
HiveDecimal convertNumberIntoHiveDecimal(Number number) { HiveDecimal convertNumberIntoHiveDecimal(Number number) {
BigDecimal bigDecimal = null; BigDecimal bigDecimal = null;

12
src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
View File

@ -446,6 +446,18 @@ public void testNumberTypes() throws Exception {
"decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.DECIMAL, 18, 2, "decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.DECIMAL, 18, 2,
HiveDecimal.create(new BigDecimal("2000")), HiveDecimal.create(new BigDecimal("2000")),
new BigDecimal("2000"), KeyType.NOT_A_KEY), new BigDecimal("2000"), KeyType.NOT_A_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(3),
"decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.VARCHAR, 20, 0,
new HiveVarchar("1999", 20), new BigDecimal("1999"), KeyType.NOT_A_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(4),
"decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.CHAR, 20, 0,
new HiveChar("1998", 20), new BigDecimal("1998"), KeyType.NOT_A_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(5),
"int", Types.INTEGER, HCatFieldSchema.Type.VARCHAR, 20, 0,
new HiveVarchar("2001", 20 ), new BigDecimal("2001"), KeyType.NOT_A_KEY),
HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(6),
"int", Types.INTEGER, HCatFieldSchema.Type.CHAR, 20, 0,
new HiveChar("2002", 20), new BigDecimal("2002"), KeyType.NOT_A_KEY),
}; };
List<String> addlArgsArray = new ArrayList<String>(); List<String> addlArgsArray = new ArrayList<String>();
setExtraArgs(addlArgsArray); setExtraArgs(addlArgsArray);