From 803edc27925ec6ee950b2e7fa2ffea5727248fa6 Mon Sep 17 00:00:00 2001 From: Attila Szabo Date: Mon, 20 Mar 2017 16:37:49 +0100 Subject: [PATCH] SQOOP-3152: Fix SqoopOptions#parseColumnMapping, thus Sqoop will support mappings with precisison (e.g. DECIMAL(10,5)) in the future properly (Eric Lin via Attila Szabo) --- src/docs/web/images/Thumbs.db | Bin 0 -> 4608 bytes src/java/org/apache/sqoop/SqoopOptions.java | 9 ++-- .../cloudera/sqoop/hive/TestHiveImport.java | 17 ++++++++ .../sqoop/hive/TestTableDefWriter.java | 39 +++++++++++++++++- testdata/hive/scripts/decimalMapImport.q | 17 ++++++++ 5 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 src/docs/web/images/Thumbs.db create mode 100644 testdata/hive/scripts/decimalMapImport.q diff --git a/src/docs/web/images/Thumbs.db b/src/docs/web/images/Thumbs.db new file mode 100644 index 0000000000000000000000000000000000000000..bc9428cf55e6920250a34a86fa01a39f2612e246 GIT binary patch literal 4608 zcmeH}2{hDQAII;Q8Dp47MrJJK9WfG3*-8r!ktIAOM6{4C+0sbX(Im2^EQO+S)YuzB zb`{z6Bt$~lvQ`R}oq6w6Pw#V_(>a~qbKY~F_w<|dx!eEV|NVdO_x|sHnx5c$lX67* zz_L&UNU+4h0?xm+BVd`$=imc?0~XlzB^HbIr4ayF{1yM8693Yv(9usz2> z*loY!|9b_DfGhmD12Lclj>FsycthuZJcw||{Ix>g&W7&{-?6>n%%j8>1QK4~!S$t+ z0@w_cfF(?Az)@fe6oEWYfay^n2e!cSW{4Fme=lSJ2jZ{!+4=YSEpJ!(I{)(apQ-=M z{690#_s0A;^Rqo;pW|##mcRe(CBj0uAv_QqQBJa+vw8uap0=(w zKp?)p*sLzF6QGbt6cU9&p)gns8iVD*!rv4a=?Im3O;zEqZ{?3*A^oSZl=91({j z3h?ss3a}GifiJ`GKXj9Ffaijzf*cYd4mj`#Bp$)4V(%#g8b0l}2ZnVHBnr)m;lgtB zzz+F*@U%c8IZ#M68U=Sd!Vg{tC_I{fjl4SNN<%A*I8{KwKPrh!LL;YwVAMQLR2geL+?A{GH*YbLQ&Q8?Gjj9p<`)ze6_-4$tg5c5t*dWndD`0c zto?aMXJ7xTfx)5SkxmHZpO1E{ zMEj=idES4~WyAG|Lo(-$%;tepKMA?i*iD%TkKb2lte!Gja*t6Mc?drhhfjGbke$|xcj3)`UFI+HE$=L-<4jxf+zo*Ou7>+O9jF$=} zJvOf=dUk2terXW1Zg0A1LmI_IgHeZzLKK)Kjg_=FkPxbBqdkO86TOzA1V6)8isH7; zjb|etec<3eyav(Slu0xsJLfWeZ=BDv@UmEv<#n1AHnKK-`00+y_$_MK;UXTlFzgWy z#t3thUXoqd3XSVphKK1D26?JFtInneq>Z+kI@wRzmgvc5(0w`>PjxQS?AqSnH3$%S z{47ziLTxH}&UI%{*IFj!xWaZj;UJ&zfg@*hhFcCC)^(gt$6as|?wky?5xIs8I-!(a zfk9p5`u)vE%qxpH-%J|eSzk`*P|PXiS#B}%+X&N`mv^>3yqQPZfz9SvjK06#;k4lW zl5x}0!Q|L%nR;ZMs_}_^n$5e+#ug{Z)|51KYm#+~cK(p{kyJyen*}!t>FqE4YV^Zn z1u@$%_oqr4cN^%`KN59s>*#(-7?@KWp`Y_nO7iYg!#c)Bg^CC4EDXQUfK@nHgBX|+ z8!GvIWB2eMzD^r7KyI}Qt=0>t(2{zb-HA6;<^osvhFiE< zl>rKctAh{2BW!8{WSsY+oxR%4^K)7TymKOhh2LDGhK=e<`o+&?$GxCYuJ^dJKz8dw zc2UAx&&mAkceLUG`C0nL$5|e4yDpTTP99DgVEbSPGP zzH!oFGB0QPC^_+@WdaM7l@!~B>X$_qv{~p&hHc6p4E13Musda5t5Fsj|hOUw`Ku{X@=3yF%fajgr2R zq)Ja?eZ@L&CAUAvC`>7%soaLQZ-*21dr)>Kv4G~R)YOf|tAe}D%ba?PG~-VONDf}J z5ts8;)Wn!Y>{p*wom{xeGaVkqsI!UWG!+t?&X#`=VLD@da9*j=&$#l*_1H+EKc|=% ziKK1)+WA|{LyC@JIVe?{RG)n5jIbvIb15`co>eN8*}l9MLpttO$5KRUcjEALAl;!S zq?`AtT5fMi(A;L{nz^&ODj{WqsPp7KucX8pS_mYWNPiQz1)GVkT1!6W#e}rS+&a>o zj~rHqG_K%OyP>;>BqmyF77?dx?mw^fZs9ixL3+?7q7?c17Jc%i)@b{I!~^k_X5&FI zr$j#mS0*I*rkSa@Ri}=Ij8sO}z1eDa#2dw@KH?Pox+On5HR(MII9Lf3x5@hYEq1X$ z(87aR*}4I`y+KFkqrT*jiT4qjR7Yyf(IDY*{sy@%-ns93_kcP~2*$FT1u{R5$-Slv zZ7%gQj~b&Xram4%z&KxEKFD1^6^uH5!9)4>-T}UF3#*~rT33q=5|%Ud8RX7ScS~G@ zn6@hjmrH3K3=7S|Rl-eUD%B!+;#3J@oPEsLl*b8&@eTJ%0z)~0sNDLd)ZOal^vc=1 jVR^6Oe)}g|Je%FaMuoCMPb=RuXLZ284Zq?az5@RSrQj25 literal 0 HcmV?d00001 diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index b33b54b6..caf95f68 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -1322,11 +1322,14 @@ public String getPasswordAlias() { public void setPasswordAlias(String alias) { this.passwordAlias = alias; } + protected void parseColumnMapping(String mapping, Properties output) { output.clear(); - String[] maps = mapping.split(","); + // replace (xx,xx) with (xx#xx), so that we can just split by "," afterwards + String[] maps = mapping.replaceAll("\\(([0-9]+),([0-9]+)\\)", "($1#$2)").split(","); + for(String map : maps) { String[] details = map.split("="); if (details.length != 2) { @@ -1336,8 +1339,8 @@ protected void parseColumnMapping(String mapping, try { output.put( - URLDecoder.decode(details[0], "UTF-8"), - URLDecoder.decode(details[1], "UTF-8")); + URLDecoder.decode(details[0].replace("#", ","), "UTF-8"), + URLDecoder.decode(details[1].replace("#", ","), "UTF-8")); } catch (UnsupportedEncodingException e) { throw new IllegalArgumentException("Encoding not supported. " + "Column mapping should be UTF-8 encoding."); diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java index 6f13fe23..33e0cc41 100644 --- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java +++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java @@ -499,6 +499,23 @@ public void testNumeric() throws IOException { getArgv(false, null), new ImportTool()); } + /** Test that DECIMALS using --map-column-hive option maps can run without issues. */ + @Test + public void testDecimalMapColumnHive() throws IOException { + final String TABLE_NAME = "DECIMAL_MAP_HIVE_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(2); + String [] types = { "NUMERIC", "CHAR(64)" }; + String [] vals = { "12343.14159", "'foo'" }; + + ArrayList args = new ArrayList(); + args.add("--map-column-hive"); + args.add(BASE_COL_NAME + "0=DECIMAL(10,10)"); + + runImportTest(TABLE_NAME, types, vals, "decimalMapImport.q", + getArgv(false, args.toArray(new String[args.size()])), new ImportTool()); + } + /** If bin/hive returns an error exit status, we should get an IOException. */ @Test public void testHiveExitFails() throws IOException { diff --git a/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java b/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java index 6af12da1..dbf0dde8 100644 --- a/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java +++ b/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java @@ -180,7 +180,7 @@ public void testLzoSplitting() throws Exception { } @Test - public void testUserMapping() throws Exception { + public void testUserMappingNoDecimal() throws Exception { String[] args = { "--map-column-hive", "id=STRING,value=INTEGER", }; @@ -206,6 +206,43 @@ public void testUserMapping() throws Exception { assertFalse(createTable.contains("`value` STRING")); } + @Test + public void testUserMappingWithDecimal() throws Exception { + String[] args = { + "--map-column-hive", "id=STRING,value2=DECIMAL(13,5),value1=INTEGER," + + "value3=DECIMAL(4,5),value4=VARCHAR(255)", + }; + Configuration conf = new Configuration(); + SqoopOptions options = + new ImportTool().parseArguments(args, null, null, false); + TableDefWriter writer = new TableDefWriter(options, + null, HsqldbTestServer.getTableName(), "outputTable", conf, false); + + Map colTypes = new SqlTypeMap(); + colTypes.put("id", Types.INTEGER); + colTypes.put("value1", Types.VARCHAR); + colTypes.put("value2", Types.DOUBLE); + colTypes.put("value3", Types.FLOAT); + colTypes.put("value4", Types.CHAR); + writer.setColumnTypes(colTypes); + + String createTable = writer.getCreateTableStmt(); + + assertNotNull(createTable); + + assertTrue(createTable.contains("`id` STRING")); + assertTrue(createTable.contains("`value1` INTEGER")); + assertTrue(createTable.contains("`value2` DECIMAL(13,5)")); + assertTrue(createTable.contains("`value3` DECIMAL(4,5)")); + assertTrue(createTable.contains("`value4` VARCHAR(255)")); + + assertFalse(createTable.contains("`id` INTEGER")); + assertFalse(createTable.contains("`value1` STRING")); + assertFalse(createTable.contains("`value2` DOUBLE")); + assertFalse(createTable.contains("`value3` FLOAT")); + assertFalse(createTable.contains("`value4` CHAR")); + } + @Test public void testUserMappingFailWhenCantBeApplied() throws Exception { String[] args = { diff --git a/testdata/hive/scripts/decimalMapImport.q b/testdata/hive/scripts/decimalMapImport.q new file mode 100644 index 00000000..8f05d585 --- /dev/null +++ b/testdata/hive/scripts/decimalMapImport.q @@ -0,0 +1,17 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +CREATE TABLE IF NOT EXISTS `DECIMAL_MAP_HIVE_IMPORT` ( `DATA_COL0` DECIMAL(10, 10), `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/DECIMAL_MAP_HIVE_IMPORT' INTO TABLE `DECIMAL_MAP_HIVE_IMPORT`;