diff --git a/src/docs/web/images/Thumbs.db b/src/docs/web/images/Thumbs.db new file mode 100644 index 00000000..bc9428cf Binary files /dev/null and b/src/docs/web/images/Thumbs.db differ diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index b33b54b6..caf95f68 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -1322,11 +1322,14 @@ public String getPasswordAlias() { public void setPasswordAlias(String alias) { this.passwordAlias = alias; } + protected void parseColumnMapping(String mapping, Properties output) { output.clear(); - String[] maps = mapping.split(","); + // replace (xx,xx) with (xx#xx), so that we can just split by "," afterwards + String[] maps = mapping.replaceAll("\\(([0-9]+),([0-9]+)\\)", "($1#$2)").split(","); + for(String map : maps) { String[] details = map.split("="); if (details.length != 2) { @@ -1336,8 +1339,8 @@ protected void parseColumnMapping(String mapping, try { output.put( - URLDecoder.decode(details[0], "UTF-8"), - URLDecoder.decode(details[1], "UTF-8")); + URLDecoder.decode(details[0].replace("#", ","), "UTF-8"), + URLDecoder.decode(details[1].replace("#", ","), "UTF-8")); } catch (UnsupportedEncodingException e) { throw new IllegalArgumentException("Encoding not supported. " + "Column mapping should be UTF-8 encoding."); diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java index 6f13fe23..33e0cc41 100644 --- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java +++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java @@ -499,6 +499,23 @@ public void testNumeric() throws IOException { getArgv(false, null), new ImportTool()); } + /** Test that DECIMALS using --map-column-hive option maps can run without issues. */ + @Test + public void testDecimalMapColumnHive() throws IOException { + final String TABLE_NAME = "DECIMAL_MAP_HIVE_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(2); + String [] types = { "NUMERIC", "CHAR(64)" }; + String [] vals = { "12343.14159", "'foo'" }; + + ArrayList args = new ArrayList(); + args.add("--map-column-hive"); + args.add(BASE_COL_NAME + "0=DECIMAL(10,10)"); + + runImportTest(TABLE_NAME, types, vals, "decimalMapImport.q", + getArgv(false, args.toArray(new String[args.size()])), new ImportTool()); + } + /** If bin/hive returns an error exit status, we should get an IOException. */ @Test public void testHiveExitFails() throws IOException { diff --git a/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java b/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java index 6af12da1..dbf0dde8 100644 --- a/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java +++ b/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java @@ -180,7 +180,7 @@ public void testLzoSplitting() throws Exception { } @Test - public void testUserMapping() throws Exception { + public void testUserMappingNoDecimal() throws Exception { String[] args = { "--map-column-hive", "id=STRING,value=INTEGER", }; @@ -206,6 +206,43 @@ public void testUserMapping() throws Exception { assertFalse(createTable.contains("`value` STRING")); } + @Test + public void testUserMappingWithDecimal() throws Exception { + String[] args = { + "--map-column-hive", "id=STRING,value2=DECIMAL(13,5),value1=INTEGER," + + "value3=DECIMAL(4,5),value4=VARCHAR(255)", + }; + Configuration conf = new Configuration(); + SqoopOptions options = + new ImportTool().parseArguments(args, null, null, false); + TableDefWriter writer = new TableDefWriter(options, + null, HsqldbTestServer.getTableName(), "outputTable", conf, false); + + Map colTypes = new SqlTypeMap(); + colTypes.put("id", Types.INTEGER); + colTypes.put("value1", Types.VARCHAR); + colTypes.put("value2", Types.DOUBLE); + colTypes.put("value3", Types.FLOAT); + colTypes.put("value4", Types.CHAR); + writer.setColumnTypes(colTypes); + + String createTable = writer.getCreateTableStmt(); + + assertNotNull(createTable); + + assertTrue(createTable.contains("`id` STRING")); + assertTrue(createTable.contains("`value1` INTEGER")); + assertTrue(createTable.contains("`value2` DECIMAL(13,5)")); + assertTrue(createTable.contains("`value3` DECIMAL(4,5)")); + assertTrue(createTable.contains("`value4` VARCHAR(255)")); + + assertFalse(createTable.contains("`id` INTEGER")); + assertFalse(createTable.contains("`value1` STRING")); + assertFalse(createTable.contains("`value2` DOUBLE")); + assertFalse(createTable.contains("`value3` FLOAT")); + assertFalse(createTable.contains("`value4` CHAR")); + } + @Test public void testUserMappingFailWhenCantBeApplied() throws Exception { String[] args = { diff --git a/testdata/hive/scripts/decimalMapImport.q b/testdata/hive/scripts/decimalMapImport.q new file mode 100644 index 00000000..8f05d585 --- /dev/null +++ b/testdata/hive/scripts/decimalMapImport.q @@ -0,0 +1,17 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +CREATE TABLE IF NOT EXISTS `DECIMAL_MAP_HIVE_IMPORT` ( `DATA_COL0` DECIMAL(10, 10), `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/DECIMAL_MAP_HIVE_IMPORT' INTO TABLE `DECIMAL_MAP_HIVE_IMPORT`;