github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-04 23:29:44 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-2828: Sqoop2: AvroIntermediateDataFormat should read Decimals as BigDecimals instead of Strings

Browse Source 
(Abraham Fine via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2016-02-10 17:03:39 -08:00
parent e64598af41
commit edb42dbdc2
2 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java
View File

@ -352,7 +352,7 @@ public String toCSV(GenericRecord record) {
break;
case DECIMAL:
// stored as string
csvString.append(toCSVDecimal(obj));
csvString.append(toCSVDecimal(new BigDecimal(obj.toString())));
break;
case DATE:
// stored as long
@ -417,9 +417,11 @@ public Object[] toObject(GenericRecord record) {
// stored as enum symbol
case TEXT:
// stored as UTF8
object[nameIndex] = obj.toString();
break;
case DECIMAL:
// stored as string
object[nameIndex] = obj.toString();
object[nameIndex] = new BigDecimal(obj.toString());
break;
case BINARY:
case UNKNOWN:

22
connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java
View File

@ -35,12 +35,14 @@
import org.apache.sqoop.schema.type.Binary;
import org.apache.sqoop.schema.type.Bit;
import org.apache.sqoop.schema.type.Column;
import org.apache.sqoop.schema.type.Decimal;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.Text;
import org.joda.time.LocalDateTime;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@ -80,7 +82,7 @@ private void createAvroIDF() {
enumCol = new org.apache.sqoop.schema.type.Enum("seven").setOptions(options);
sqoopSchema
.addColumn(new FixedPoint("one", 8L, true))
.addColumn(new FixedPoint("two", 2L, true))
.addColumn(new Decimal("two", 4, 2))
.addColumn(new Text("three"))
.addColumn(new Text("four"))
.addColumn(new Binary("five"))
@ -105,7 +107,7 @@ private void createAvroIDF() {
@Test
public void testInputAsCSVTextInAndDataOut() {
String csvText = "10,34,'54','random data',"
String csvText = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A)
+ "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + ","
+ csvDate + ",13.44," + csvSet;
@ -116,7 +118,7 @@ public void testInputAsCSVTextInAndDataOut() {
@Test
public void testInputAsCSVTextInAndObjectArrayOut() {
String csvText = "10,34,'54','random data',"
String csvText = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A)
+ "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + ","
+ csvDate + ",13.44," + csvSet;
@ -129,7 +131,7 @@ public void testInputAsCSVTextInAndObjectArrayOut() {
private void assertObjectArray() {
Object[] out = dataFormat.getObjectData();
assertEquals(10L, out[0]);
assertEquals(34, out[1]);
assertEquals(new BigDecimal("34.56"), out[1]);
assertEquals("54", out[2]);
assertEquals("random data", out[3]);
assertEquals(-112, ((byte[]) out[4])[0]);
@ -173,7 +175,7 @@ private void assertObjectArray() {
@Test
public void testInputAsCSVTextInCSVTextOut() {
String csvText = "10,34,'54','random data',"
String csvText = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A)
+ "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + ","
+ csvDate + ",13.44," + csvSet;
@ -184,7 +186,7 @@ public void testInputAsCSVTextInCSVTextOut() {
private GenericRecord createAvroGenericRecord() {
GenericRecord avroObject = new GenericData.Record(avroSchema);
avroObject.put("one", 10L);
avroObject.put("two", 34);
avroObject.put("two", "34.56");
avroObject.put("three", new Utf8("54"));
avroObject.put("four", new Utf8("random data"));
// store byte array in byte buffer
@ -235,7 +237,7 @@ private GenericRecord createAvroGenericRecord() {
@Test
public void testInputAsDataInAndCSVOut() {
String csvExpected = "10,34,'54','random data',"
String csvExpected = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A)
+ "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + ","
+ csvDate + ",13.44," + csvSet;
@ -260,7 +262,7 @@ public void testInputAsDataInAndDataOut() {
private Object[] createObjectArray() {
Object[] out = new Object[15];
out[0] = 10L;
out[1] = 34;
out[1] = new BigDecimal("34.56");
out[2] = "54";
out[3] = "random data";
out[4] = new byte[] { (byte) -112, (byte) 54 };
@ -322,7 +324,7 @@ public void testInputAsObjectArrayInAndDataOut() {
public void testInputAsObjectArrayInAndCSVOut() {
Object[] out = createObjectArray();
dataFormat.setObjectData(out);
String csvText = "10,34,'54','random data',"
String csvText = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A)
+ "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + ","
+ csvDate + ",13.44," + csvSet;
@ -339,7 +341,7 @@ public void testInputAsObjectArrayInAndObjectArrayOut() {
// **************test cases for empty and null schema*******************
@Test(expectedExceptions = SqoopException.class)
public void testEmptySchema() {
String testData = "10,34,'54','random data',"
String testData = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'\\n'";
// no coumns
Schema schema = new Schema("Test");