From edb42dbdc29f3834a7bb4eea291e15c8fce77053 Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Wed, 10 Feb 2016 17:03:39 -0800 Subject: [PATCH] SQOOP-2828: Sqoop2: AvroIntermediateDataFormat should read Decimals as BigDecimals instead of Strings (Abraham Fine via Jarek Jarcec Cecho) --- .../idf/AVROIntermediateDataFormat.java | 6 +++-- .../idf/TestAVROIntermediateDataFormat.java | 22 ++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java index e409fc12..b55f7a00 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java @@ -352,7 +352,7 @@ public String toCSV(GenericRecord record) { break; case DECIMAL: // stored as string - csvString.append(toCSVDecimal(obj)); + csvString.append(toCSVDecimal(new BigDecimal(obj.toString()))); break; case DATE: // stored as long @@ -417,9 +417,11 @@ public Object[] toObject(GenericRecord record) { // stored as enum symbol case TEXT: // stored as UTF8 + object[nameIndex] = obj.toString(); + break; case DECIMAL: // stored as string - object[nameIndex] = obj.toString(); + object[nameIndex] = new BigDecimal(obj.toString()); break; case BINARY: case UNKNOWN: diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java index 84757207..3c4d7ded 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java @@ -35,12 +35,14 @@ import org.apache.sqoop.schema.type.Binary; import org.apache.sqoop.schema.type.Bit; import org.apache.sqoop.schema.type.Column; +import org.apache.sqoop.schema.type.Decimal; import org.apache.sqoop.schema.type.FixedPoint; import org.apache.sqoop.schema.type.Text; import org.joda.time.LocalDateTime; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -80,7 +82,7 @@ private void createAvroIDF() { enumCol = new org.apache.sqoop.schema.type.Enum("seven").setOptions(options); sqoopSchema .addColumn(new FixedPoint("one", 8L, true)) - .addColumn(new FixedPoint("two", 2L, true)) + .addColumn(new Decimal("two", 4, 2)) .addColumn(new Text("three")) .addColumn(new Text("four")) .addColumn(new Binary("five")) @@ -105,7 +107,7 @@ private void createAvroIDF() { @Test public void testInputAsCSVTextInAndDataOut() { - String csvText = "10,34,'54','random data'," + String csvText = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) + "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + "," + csvDate + ",13.44," + csvSet; @@ -116,7 +118,7 @@ public void testInputAsCSVTextInAndDataOut() { @Test public void testInputAsCSVTextInAndObjectArrayOut() { - String csvText = "10,34,'54','random data'," + String csvText = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) + "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + "," + csvDate + ",13.44," + csvSet; @@ -129,7 +131,7 @@ public void testInputAsCSVTextInAndObjectArrayOut() { private void assertObjectArray() { Object[] out = dataFormat.getObjectData(); assertEquals(10L, out[0]); - assertEquals(34, out[1]); + assertEquals(new BigDecimal("34.56"), out[1]); assertEquals("54", out[2]); assertEquals("random data", out[3]); assertEquals(-112, ((byte[]) out[4])[0]); @@ -173,7 +175,7 @@ private void assertObjectArray() { @Test public void testInputAsCSVTextInCSVTextOut() { - String csvText = "10,34,'54','random data'," + String csvText = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) + "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + "," + csvDate + ",13.44," + csvSet; @@ -184,7 +186,7 @@ public void testInputAsCSVTextInCSVTextOut() { private GenericRecord createAvroGenericRecord() { GenericRecord avroObject = new GenericData.Record(avroSchema); avroObject.put("one", 10L); - avroObject.put("two", 34); + avroObject.put("two", "34.56"); avroObject.put("three", new Utf8("54")); avroObject.put("four", new Utf8("random data")); // store byte array in byte buffer @@ -235,7 +237,7 @@ private GenericRecord createAvroGenericRecord() { @Test public void testInputAsDataInAndCSVOut() { - String csvExpected = "10,34,'54','random data'," + String csvExpected = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) + "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + "," + csvDate + ",13.44," + csvSet; @@ -260,7 +262,7 @@ public void testInputAsDataInAndDataOut() { private Object[] createObjectArray() { Object[] out = new Object[15]; out[0] = 10L; - out[1] = 34; + out[1] = new BigDecimal("34.56"); out[2] = "54"; out[3] = "random data"; out[4] = new byte[] { (byte) -112, (byte) 54 }; @@ -322,7 +324,7 @@ public void testInputAsObjectArrayInAndDataOut() { public void testInputAsObjectArrayInAndCSVOut() { Object[] out = createObjectArray(); dataFormat.setObjectData(out); - String csvText = "10,34,'54','random data'," + String csvText = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) + "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + "," + csvDate + ",13.44," + csvSet; @@ -339,7 +341,7 @@ public void testInputAsObjectArrayInAndObjectArrayOut() { // **************test cases for empty and null schema******************* @Test(expectedExceptions = SqoopException.class) public void testEmptySchema() { - String testData = "10,34,'54','random data'," + String testData = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'\\n'"; // no coumns Schema schema = new Schema("Test");