From fd3634dc3c6568c5d1f3515e8f47b474805b5809 Mon Sep 17 00:00:00 2001 From: Arvind Prabhakar Date: Tue, 4 Oct 2011 01:12:13 +0000 Subject: [PATCH] SQOOP-341. Support for unsigned integers with MySQL. (Alex Newman via Arvind Prabhakar) git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1178661 13f79535-47bb-0310-9956-ffa450edef68 --- .../cloudera/sqoop/manager/SqlManager.java | 6 +++++ .../mapreduce/db/DataDrivenDBInputFormat.java | 7 +++++ .../sqoop/manager/DirectMySQLTest.java | 27 ++++++++++--------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/java/com/cloudera/sqoop/manager/SqlManager.java b/src/java/com/cloudera/sqoop/manager/SqlManager.java index 687dae77..51ca65bb 100644 --- a/src/java/com/cloudera/sqoop/manager/SqlManager.java +++ b/src/java/com/cloudera/sqoop/manager/SqlManager.java @@ -206,6 +206,12 @@ protected Map getColumnTypesForRawQuery(String stmt) { ResultSetMetaData metadata = results.getMetaData(); for (int i = 1; i < cols + 1; i++) { int typeId = metadata.getColumnType(i); + // If we have an unsigned int we need to make extra room by + // plopping it into a bigint + if (typeId == Types.INTEGER && !metadata.isSigned(i)){ + typeId = Types.BIGINT; + } + String colName = metadata.getColumnName(i); if (colName == null || colName.equals("")) { colName = metadata.getColumnLabel(i); diff --git a/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java b/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java index 06056ddd..b6c7f421 100644 --- a/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java +++ b/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java @@ -199,6 +199,13 @@ public List getSplits(JobContext job) throws IOException { // for interpolating split points (i.e., numeric splits, text splits, // dates, etc.) int sqlDataType = results.getMetaData().getColumnType(1); + boolean isSigned = results.getMetaData().isSigned(1); + + // MySQL has an unsigned integer which we need to allocate space for + if (sqlDataType == Types.INTEGER && !isSigned){ + sqlDataType = Types.BIGINT; + } + DBSplitter splitter = getSplitter(sqlDataType); if (null == splitter) { throw new IOException("Unknown SQL data type: " + sqlDataType); diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java index 6f9b1363..897f3079 100644 --- a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java +++ b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java @@ -105,16 +105,17 @@ public void setUp() { st.executeUpdate("CREATE TABLE " + getTableName() + " (" + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, " + "name VARCHAR(24) NOT NULL, " + + "overly_large_number INT UNSIGNED," + "start_date DATE, " + "salary FLOAT, " + "dept VARCHAR(32))"); st.executeUpdate("INSERT INTO " + getTableName() + " VALUES(" - + "NULL,'Aaron','2009-05-14',1000000.00,'engineering')"); + + "NULL,'Aaron',0,'2009-05-14',1000000.00,'engineering')"); st.executeUpdate("INSERT INTO " + getTableName() + " VALUES(" - + "NULL,'Bob','2009-04-20',400.00,'sales')"); + + "NULL,'Bob',100,'2009-04-20',400.00,'sales')"); st.executeUpdate("INSERT INTO " + getTableName() + " VALUES(" - + "NULL,'Fred','2009-01-23',15.00,'marketing')"); + + "NULL,'Fred',4000000000,'2009-01-23',15.00,'marketing')"); connection.commit(); } catch (SQLException sqlE) { LOG.error("Encountered SQL Exception: " + sqlE); @@ -231,8 +232,8 @@ private void doImport(boolean mysqlOutputDelims, boolean isDirect, public void testDirectBulkImportWithDefaultDelims() throws IOException { // no quoting of strings allowed. String [] expectedResults = { - "2,Bob,2009-04-20,400,sales", - "3,Fred,2009-01-23,15,marketing", + "2,Bob,100,2009-04-20,400,sales", + "3,Fred,4000000000,2009-01-23,15,marketing", }; doImport(false, true, getTableName(), expectedResults, null); @@ -242,8 +243,8 @@ public void testDirectBulkImportWithDefaultDelims() throws IOException { public void testWithExtraParams() throws IOException { // no quoting of strings allowed. String [] expectedResults = { - "2,Bob,2009-04-20,400,sales", - "3,Fred,2009-01-23,15,marketing", + "2,Bob,100,2009-04-20,400,sales", + "3,Fred,4000000000,2009-01-23,15,marketing", }; String [] extraArgs = { "--", "--lock-tables" }; @@ -255,8 +256,8 @@ public void testWithExtraParams() throws IOException { public void testMultiMappers() throws IOException { // no quoting of strings allowed. String [] expectedResults = { - "2,Bob,2009-04-20,400,sales", - "3,Fred,2009-01-23,15,marketing", + "2,Bob,100,2009-04-20,400,sales", + "3,Fred,4000000000,2009-01-23,15,marketing", }; String [] extraArgs = { "-m", "2" }; @@ -297,8 +298,8 @@ public void testDirectColumnSubset() throws IOException { public void testDirectBulkImportWithMySQLQuotes() throws IOException { // mysql quotes all string-based output. String [] expectedResults = { - "2,'Bob','2009-04-20',400,'sales'", - "3,'Fred','2009-01-23',15,'marketing'", + "2,'Bob',100,'2009-04-20',400,'sales'", + "3,'Fred',4000000000,'2009-01-23',15,'marketing'", }; doImport(true, true, getTableName(), expectedResults, null); @@ -307,8 +308,8 @@ public void testDirectBulkImportWithMySQLQuotes() throws IOException { @Test public void testMySQLJdbcImport() throws IOException { String [] expectedResults = { - "2,Bob,2009-04-20,400.0,sales", - "3,Fred,2009-01-23,15.0,marketing", + "2,Bob,100,2009-04-20,400.0,sales", + "3,Fred,4000000000,2009-01-23,15.0,marketing", }; doImport(false, false, getTableName(), expectedResults, null);