SQOOP-341. Support for unsigned integers with MySQL.

(Alex Newman via Arvind Prabhakar)


git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1178661 13f79535-47bb-0310-9956-ffa450edef68

src/java/com/cloudera/sqoop/manager/SqlManager.java

@@ -206,6 +206,12 @@ protected Map<String, Integer> getColumnTypesForRawQuery(String stmt) {
       ResultSetMetaData metadata = results.getMetaData();
       for (int i = 1; i < cols + 1; i++) {
         int typeId = metadata.getColumnType(i);
+        // If we have an unsigned int we need to make extra room by
+        // plopping it into a bigint
+        if (typeId == Types.INTEGER &&  !metadata.isSigned(i)){
+            typeId = Types.BIGINT;
+        }
+
         String colName = metadata.getColumnName(i);
         if (colName == null || colName.equals("")) {
           colName = metadata.getColumnLabel(i);

src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java

@@ -199,6 +199,13 @@ public List<InputSplit> getSplits(JobContext job) throws IOException {
       // for interpolating split points (i.e., numeric splits, text splits,
       // dates, etc.)
       int sqlDataType = results.getMetaData().getColumnType(1);
+      boolean isSigned = results.getMetaData().isSigned(1);
+
+      // MySQL has an unsigned integer which we need to allocate space for
+      if (sqlDataType == Types.INTEGER && !isSigned){
+          sqlDataType = Types.BIGINT;
+      }
+
       DBSplitter splitter = getSplitter(sqlDataType);
       if (null == splitter) {
         throw new IOException("Unknown SQL data type: " + sqlDataType);

src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java

@@ -105,16 +105,17 @@ public void setUp() {
       st.executeUpdate("CREATE TABLE " + getTableName() + " ("
           + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
           + "name VARCHAR(24) NOT NULL, "
+          + "overly_large_number INT UNSIGNED,"
           + "start_date DATE, "
           + "salary FLOAT, "
           + "dept VARCHAR(32))");
 
       st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
-          + "NULL,'Aaron','2009-05-14',1000000.00,'engineering')");
+          + "NULL,'Aaron',0,'2009-05-14',1000000.00,'engineering')");
       st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
-          + "NULL,'Bob','2009-04-20',400.00,'sales')");
+          + "NULL,'Bob',100,'2009-04-20',400.00,'sales')");
       st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
-          + "NULL,'Fred','2009-01-23',15.00,'marketing')");
+          + "NULL,'Fred',4000000000,'2009-01-23',15.00,'marketing')");
       connection.commit();
     } catch (SQLException sqlE) {
       LOG.error("Encountered SQL Exception: " + sqlE);
@@ -231,8 +232,8 @@ private void doImport(boolean mysqlOutputDelims, boolean isDirect,
   public void testDirectBulkImportWithDefaultDelims() throws IOException {
     // no quoting of strings allowed.
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400,sales",
+      "2,Bob,100,2009-04-20,400,sales",
-      "3,Fred,2009-01-23,15,marketing",
+      "3,Fred,4000000000,2009-01-23,15,marketing",
     };
 
     doImport(false, true, getTableName(), expectedResults, null);
@@ -242,8 +243,8 @@ public void testDirectBulkImportWithDefaultDelims() throws IOException {
   public void testWithExtraParams() throws IOException {
     // no quoting of strings allowed.
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400,sales",
+      "2,Bob,100,2009-04-20,400,sales",
-      "3,Fred,2009-01-23,15,marketing",
+      "3,Fred,4000000000,2009-01-23,15,marketing",
     };
 
     String [] extraArgs = { "--", "--lock-tables" };
@@ -255,8 +256,8 @@ public void testWithExtraParams() throws IOException {
   public void testMultiMappers() throws IOException {
     // no quoting of strings allowed.
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400,sales",
+      "2,Bob,100,2009-04-20,400,sales",
-      "3,Fred,2009-01-23,15,marketing",
+      "3,Fred,4000000000,2009-01-23,15,marketing",
     };
 
     String [] extraArgs = { "-m", "2" };
@@ -297,8 +298,8 @@ public void testDirectColumnSubset() throws IOException {
   public void testDirectBulkImportWithMySQLQuotes() throws IOException {
     // mysql quotes all string-based output.
     String [] expectedResults = {
-      "2,'Bob','2009-04-20',400,'sales'",
+      "2,'Bob',100,'2009-04-20',400,'sales'",
-      "3,'Fred','2009-01-23',15,'marketing'",
+      "3,'Fred',4000000000,'2009-01-23',15,'marketing'",
     };
 
     doImport(true, true, getTableName(), expectedResults, null);
@@ -307,8 +308,8 @@ public void testDirectBulkImportWithMySQLQuotes() throws IOException {
   @Test
   public void testMySQLJdbcImport() throws IOException {
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400.0,sales",
+      "2,Bob,100,2009-04-20,400.0,sales",
-      "3,Fred,2009-01-23,15.0,marketing",
+      "3,Fred,4000000000,2009-01-23,15.0,marketing",
     };
 
     doImport(false, false, getTableName(), expectedResults, null);