github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-12 15:01:45 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1819: Support Enum in CSVIDF ( + add unit tests)

Browse Source 
(Veena Basavaraj via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2014-12-01 17:38:53 -08:00
parent d68c05d3ad
commit 38f2894ed6
2 changed files with 29 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
View File

@ -128,7 +128,7 @@ public void setSchema(Schema schema) {
List<Column> columns = schema.getColumns();
int i = 0;
for (Column col : columns) {
if (col.getType() == ColumnType.TEXT) {
if (isColumnStringType(col) ) {
stringTypeColumnIndices.add(i);
} else if (col.getType() == ColumnType.BINARY) {
byteTypeColumnIndices.add(i);
@ -232,6 +232,7 @@ private Object parseCSVStringArrayElement(String fieldString, Column column) {
Object returnValue = null;
switch (column.getType()) {
case ENUM:
case TEXT:
returnValue = unescapeString(fieldString);
break;
@ -453,6 +454,11 @@ private boolean isColumnListType(Column listType) {
return listType.getType().equals(ColumnType.ARRAY) || listType.getType().equals(ColumnType.SET);
}
private boolean isColumnStringType(Column stringType) {
return stringType.getType().equals(ColumnType.TEXT)
|| stringType.getType().equals(ColumnType.ENUM);
}
private String escapeByteArrays(byte[] bytes) {
try {
return escapeString(new String(bytes, BYTE_FIELD_CHARSET));

34
connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
View File

@ -99,7 +99,7 @@ public void testEmptyInputAsCSVTextInObjectArrayOut() {
@Test
public void testInputAsCSVTextInCSVTextOut() {
String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
String testData = "'ENUM',10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
+ ",'" + String.valueOf(0x0A) + "'";
dataFormat.setTextData(testData);
assertEquals(testData, dataFormat.getTextData());
@ -110,14 +110,15 @@ public void testInputAsCSVTextInObjectOut() {
//byte[0] = -112, byte[1] = 54 - 2's complements
String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
+ ",'\\n'";
+ ",'\\n','TEST_ENUM'";
Schema schema = new Schema("test");
schema.addColumn(new FixedPoint("1"))
.addColumn(new FixedPoint("2"))
.addColumn(new Text("3"))
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
.addColumn(new Text("6"))
.addColumn(new org.apache.sqoop.schema.type.Enum("7"));
dataFormat.setSchema(schema);
dataFormat.setTextData(testData);
@ -131,6 +132,7 @@ public void testInputAsCSVTextInObjectOut() {
assertEquals(-112, ((byte[]) out[4])[0]);
assertEquals(54, ((byte[])out[4])[1]);
assertEquals("\n", out[5].toString());
assertEquals("TEST_ENUM", out[6].toString());
}
@Test
@ -141,23 +143,25 @@ public void testInputAsObjectArayInCSVTextOut() {
.addColumn(new Text("3"))
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
.addColumn(new Text("6"))
.addColumn(new org.apache.sqoop.schema.type.Enum("7"));
dataFormat.setSchema(schema);
byte[] byteFieldData = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54};
Object[] in = new Object[6];
Object[] in = new Object[7];
in[0] = new Long(10);
in[1] = new Long(34);
in[2] = "54";
in[3] = "random data";
in[4] = byteFieldData;
in[5] = new String(new char[] { 0x0A });
in[6] = "TEST_ENUM";
dataFormat.setObjectData(in);
//byte[0] = \r byte[1] = -112, byte[1] = 54 - 2's complements
String testData = "10,34,'54','random data'," +
getByteFieldString(byteFieldData).replaceAll("\r", "\\\\r") + ",'\\n'";
getByteFieldString(byteFieldData).replaceAll("\r", "\\\\r") + ",'\\n','TEST_ENUM'";
assertEquals(testData, dataFormat.getTextData());
}
@ -171,17 +175,20 @@ public void testObjectArrayInObjectArrayOut() {
.addColumn(new Text("3"))
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
.addColumn(new Text("6"))
.addColumn(new org.apache.sqoop.schema.type.Enum("7"));
dataFormat.setSchema(schema);
Object[] in = new Object[6];
Object[] in = new Object[7];
in[0] = new Long(10);
in[1] = new Long(34);
in[2] = "54";
in[3] = "random data";
in[4] = new byte[] { (byte) -112, (byte) 54};
in[5] = new String(new char[] { 0x0A });
Object[] inCopy = new Object[6];
in[6] = "TEST_ENUM";
Object[] inCopy = new Object[7];
System.arraycopy(in,0,inCopy,0,in.length);
// Modifies the input array, so we use the copy to confirm
@ -198,23 +205,26 @@ public void testObjectArrayWithNullInCSVTextOut() {
.addColumn(new Text("3"))
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
.addColumn(new Text("6"))
.addColumn(new org.apache.sqoop.schema.type.Enum("7"));
dataFormat.setSchema(schema);
byte[] byteFieldData = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54};
Object[] in = new Object[6];
Object[] in = new Object[7];
in[0] = new Long(10);
in[1] = new Long(34);
in[2] = null;
in[3] = "random data";
in[4] = byteFieldData;
in[5] = new String(new char[] { 0x0A });
in[6] = "TEST_ENUM";
dataFormat.setObjectData(in);
//byte[0] = \r byte[1] = -112, byte[1] = 54 - 2's complements
String testData = "10,34,NULL,'random data'," +
getByteFieldString(byteFieldData).replaceAll("\r", "\\\\r") + ",'\\n'";
getByteFieldString(byteFieldData).replaceAll("\r", "\\\\r") + ",'\\n','TEST_ENUM'";
assertEquals(testData, dataFormat.getTextData());
}