diff --git a/src/java/org/apache/hadoop/sqoop/hive/TableDefWriter.java b/src/java/org/apache/hadoop/sqoop/hive/TableDefWriter.java index 28f83d43..81550afd 100644 --- a/src/java/org/apache/hadoop/sqoop/hive/TableDefWriter.java +++ b/src/java/org/apache/hadoop/sqoop/hive/TableDefWriter.java @@ -115,10 +115,10 @@ public String getCreateTableStmt() throws IOException { sb.append("COMMENT 'Imported by sqoop on " + curDateStr + "' "); } - sb.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\0"); - sb.append(Integer.toOctalString((int) options.getOutputFieldDelim())); - sb.append("' LINES TERMINATED BY '\\0"); - sb.append(Integer.toOctalString((int) options.getOutputRecordDelim())); + sb.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '"); + sb.append(getHiveOctalCharCode((int) options.getOutputFieldDelim())); + sb.append("' LINES TERMINATED BY '"); + sb.append(getHiveOctalCharCode((int) options.getOutputRecordDelim())); sb.append("' STORED AS TEXTFILE"); LOG.debug("Create statement: " + sb.toString()); @@ -170,5 +170,28 @@ public String getLoadDataStmt() throws IOException { LOG.debug("Load statement: " + sb.toString()); return sb.toString(); } + + /** + * Return a string identifying the character to use as a delimiter + * in Hive, in octal representation. + * Hive can specify delimiter characters in the form '\ooo' where + * ooo is a three-digit octal number between 000 and 177. Values + * may not be truncated ('\12' is wrong; '\012' is ok) nor may they + * be zero-prefixed (e.g., '\0177' is wrong). + * + * @param charNum the character to use as a delimiter + * @return a string of the form "\ooo" where ooo is an octal number + * in [000, 177]. + * @throws IllegalArgumentException if charNum >> 0177. + */ + static String getHiveOctalCharCode(int charNum) + throws IllegalArgumentException { + if (charNum > 0177) { + throw new IllegalArgumentException( + "Character " + charNum + " is an out-of-range delimiter"); + } + + return String.format("\\%03o", charNum); + } } diff --git a/src/test/org/apache/hadoop/sqoop/SmokeTests.java b/src/test/org/apache/hadoop/sqoop/SmokeTests.java index 366e9a2a..5f1059af 100644 --- a/src/test/org/apache/hadoop/sqoop/SmokeTests.java +++ b/src/test/org/apache/hadoop/sqoop/SmokeTests.java @@ -19,6 +19,7 @@ package org.apache.hadoop.sqoop; import org.apache.hadoop.sqoop.hive.TestHiveImport; +import org.apache.hadoop.sqoop.hive.TestTableDefWriter; import org.apache.hadoop.sqoop.io.TestSplittableBufferedWriter; import org.apache.hadoop.sqoop.lib.TestFieldFormatter; import org.apache.hadoop.sqoop.lib.TestRecordParser; @@ -59,6 +60,7 @@ public static Test suite() { suite.addTestSuite(TestParseMethods.class); suite.addTestSuite(TestConnFactory.class); suite.addTestSuite(TestSplittableBufferedWriter.class); + suite.addTestSuite(TestTableDefWriter.class); suite.addTest(MapreduceTests.suite()); return suite; diff --git a/src/test/org/apache/hadoop/sqoop/hive/TestTableDefWriter.java b/src/test/org/apache/hadoop/sqoop/hive/TestTableDefWriter.java new file mode 100644 index 00000000..e8aed692 --- /dev/null +++ b/src/test/org/apache/hadoop/sqoop/hive/TestTableDefWriter.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.sqoop.hive; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Test; + +import junit.framework.TestCase; + +/** + * Test Hive DDL statement generation. + */ +public class TestTableDefWriter extends TestCase { + + public static final Log LOG = LogFactory.getLog(TestHiveImport.class.getName()); + + + // Test getHiveOctalCharCode and expect an IllegalArgumentException. + private void expectExceptionInCharCode(int charCode) { + try { + TableDefWriter.getHiveOctalCharCode(charCode); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException iae) { + // Expected; ok. + } + } + + public void testHiveOctalCharCode() { + assertEquals("\\000", TableDefWriter.getHiveOctalCharCode(0)); + assertEquals("\\001", TableDefWriter.getHiveOctalCharCode(1)); + assertEquals("\\012", TableDefWriter.getHiveOctalCharCode((int) '\n')); + assertEquals("\\177", TableDefWriter.getHiveOctalCharCode(0177)); + + expectExceptionInCharCode(4096); + expectExceptionInCharCode(0200); + expectExceptionInCharCode(254); + } +} diff --git a/testdata/hive/scripts/customDelimImport.q b/testdata/hive/scripts/customDelimImport.q index a64166e9..1e7defb6 100644 --- a/testdata/hive/scripts/customDelimImport.q +++ b/testdata/hive/scripts/customDelimImport.q @@ -1,2 +1,2 @@ -CREATE TABLE CUSTOM_DELIM_IMPORT ( DATA_COL0 STRING, DATA_COL1 INT, DATA_COL2 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\054' LINES TERMINATED BY '\0174' STORED AS TEXTFILE; +CREATE TABLE CUSTOM_DELIM_IMPORT ( DATA_COL0 STRING, DATA_COL1 INT, DATA_COL2 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\054' LINES TERMINATED BY '\174' STORED AS TEXTFILE; LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/CUSTOM_DELIM_IMPORT' INTO TABLE CUSTOM_DELIM_IMPORT; diff --git a/testdata/hive/scripts/dateImport.q b/testdata/hive/scripts/dateImport.q index fc74ff32..223f67a7 100644 --- a/testdata/hive/scripts/dateImport.q +++ b/testdata/hive/scripts/dateImport.q @@ -1,2 +1,2 @@ -CREATE TABLE DATE_HIVE_IMPORT ( DATA_COL0 STRING, DATA_COL1 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\01' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +CREATE TABLE DATE_HIVE_IMPORT ( DATA_COL0 STRING, DATA_COL1 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/DATE_HIVE_IMPORT' INTO TABLE DATE_HIVE_IMPORT; diff --git a/testdata/hive/scripts/failingImport.q b/testdata/hive/scripts/failingImport.q index fc74ff32..223f67a7 100644 --- a/testdata/hive/scripts/failingImport.q +++ b/testdata/hive/scripts/failingImport.q @@ -1,2 +1,2 @@ -CREATE TABLE DATE_HIVE_IMPORT ( DATA_COL0 STRING, DATA_COL1 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\01' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +CREATE TABLE DATE_HIVE_IMPORT ( DATA_COL0 STRING, DATA_COL1 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/DATE_HIVE_IMPORT' INTO TABLE DATE_HIVE_IMPORT; diff --git a/testdata/hive/scripts/normalImport.q b/testdata/hive/scripts/normalImport.q index 328e37c0..76e690c9 100644 --- a/testdata/hive/scripts/normalImport.q +++ b/testdata/hive/scripts/normalImport.q @@ -1,2 +1,2 @@ -CREATE TABLE NORMAL_HIVE_IMPORT ( DATA_COL0 STRING, DATA_COL1 INT, DATA_COL2 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\01' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +CREATE TABLE NORMAL_HIVE_IMPORT ( DATA_COL0 STRING, DATA_COL1 INT, DATA_COL2 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/NORMAL_HIVE_IMPORT' INTO TABLE NORMAL_HIVE_IMPORT; diff --git a/testdata/hive/scripts/numericImport.q b/testdata/hive/scripts/numericImport.q index bb6c1546..263b2952 100644 --- a/testdata/hive/scripts/numericImport.q +++ b/testdata/hive/scripts/numericImport.q @@ -1,2 +1,2 @@ -CREATE TABLE NUMERIC_HIVE_IMPORT ( DATA_COL0 DOUBLE, DATA_COL1 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\01' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +CREATE TABLE NUMERIC_HIVE_IMPORT ( DATA_COL0 DOUBLE, DATA_COL1 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/NUMERIC_HIVE_IMPORT' INTO TABLE NUMERIC_HIVE_IMPORT;