github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 04:11:44 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-341. Support for unsigned integers with MySQL.

Browse Source 
(Alex Newman via Arvind Prabhakar)


git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1178661 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arvind Prabhakar 2011-10-04 01:12:13 +00:00
parent 7c6b8b3e7d
commit fd3634dc3c
3 changed files with 27 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/java/com/cloudera/sqoop/manager/SqlManager.java
View File

@ -206,6 +206,12 @@ protected Map<String, Integer> getColumnTypesForRawQuery(String stmt) {
ResultSetMetaData metadata = results.getMetaData();
for (int i = 1; i < cols + 1; i++) {
int typeId = metadata.getColumnType(i);
// If we have an unsigned int we need to make extra room by
// plopping it into a bigint
if (typeId == Types.INTEGER && !metadata.isSigned(i)){
typeId = Types.BIGINT;
}
String colName = metadata.getColumnName(i);
if (colName == null || colName.equals("")) {
colName = metadata.getColumnLabel(i);

7
src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
View File

@ -199,6 +199,13 @@ public List<InputSplit> getSplits(JobContext job) throws IOException {
// for interpolating split points (i.e., numeric splits, text splits,
// dates, etc.)
int sqlDataType = results.getMetaData().getColumnType(1);
boolean isSigned = results.getMetaData().isSigned(1);
// MySQL has an unsigned integer which we need to allocate space for
if (sqlDataType == Types.INTEGER && !isSigned){
sqlDataType = Types.BIGINT;
}
DBSplitter splitter = getSplitter(sqlDataType);
if (null == splitter) {
throw new IOException("Unknown SQL data type: " + sqlDataType);

27
src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
View File

@ -105,16 +105,17 @@ public void setUp() {
st.executeUpdate("CREATE TABLE " + getTableName() + " ("
+ "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
+ "name VARCHAR(24) NOT NULL, "
+ "overly_large_number INT UNSIGNED,"
+ "start_date DATE, "
+ "salary FLOAT, "
+ "dept VARCHAR(32))");
st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
+ "NULL,'Aaron','2009-05-14',1000000.00,'engineering')");
+ "NULL,'Aaron',0,'2009-05-14',1000000.00,'engineering')");
st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
+ "NULL,'Bob','2009-04-20',400.00,'sales')");
+ "NULL,'Bob',100,'2009-04-20',400.00,'sales')");
st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
+ "NULL,'Fred','2009-01-23',15.00,'marketing')");
+ "NULL,'Fred',4000000000,'2009-01-23',15.00,'marketing')");
connection.commit();
} catch (SQLException sqlE) {
LOG.error("Encountered SQL Exception: " + sqlE);
@ -231,8 +232,8 @@ private void doImport(boolean mysqlOutputDelims, boolean isDirect,
public void testDirectBulkImportWithDefaultDelims() throws IOException {
// no quoting of strings allowed.
String [] expectedResults = {
"2,Bob,2009-04-20,400,sales",
"3,Fred,2009-01-23,15,marketing",
"2,Bob,100,2009-04-20,400,sales",
"3,Fred,4000000000,2009-01-23,15,marketing",
};
doImport(false, true, getTableName(), expectedResults, null);
@ -242,8 +243,8 @@ public void testDirectBulkImportWithDefaultDelims() throws IOException {
public void testWithExtraParams() throws IOException {
// no quoting of strings allowed.
String [] expectedResults = {
"2,Bob,2009-04-20,400,sales",
"3,Fred,2009-01-23,15,marketing",
"2,Bob,100,2009-04-20,400,sales",
"3,Fred,4000000000,2009-01-23,15,marketing",
};
String [] extraArgs = { "--", "--lock-tables" };
@ -255,8 +256,8 @@ public void testWithExtraParams() throws IOException {
public void testMultiMappers() throws IOException {
// no quoting of strings allowed.
String [] expectedResults = {
"2,Bob,2009-04-20,400,sales",
"3,Fred,2009-01-23,15,marketing",
"2,Bob,100,2009-04-20,400,sales",
"3,Fred,4000000000,2009-01-23,15,marketing",
};
String [] extraArgs = { "-m", "2" };
@ -297,8 +298,8 @@ public void testDirectColumnSubset() throws IOException {
public void testDirectBulkImportWithMySQLQuotes() throws IOException {
// mysql quotes all string-based output.
String [] expectedResults = {
"2,'Bob','2009-04-20',400,'sales'",
"3,'Fred','2009-01-23',15,'marketing'",
"2,'Bob',100,'2009-04-20',400,'sales'",
"3,'Fred',4000000000,'2009-01-23',15,'marketing'",
};
doImport(true, true, getTableName(), expectedResults, null);
@ -307,8 +308,8 @@ public void testDirectBulkImportWithMySQLQuotes() throws IOException {
@Test
public void testMySQLJdbcImport() throws IOException {
String [] expectedResults = {
"2,Bob,2009-04-20,400.0,sales",
"3,Fred,2009-01-23,15.0,marketing",
"2,Bob,100,2009-04-20,400.0,sales",
"3,Fred,4000000000,2009-01-23,15.0,marketing",
};
doImport(false, false, getTableName(), expectedResults, null);