From 24d08185c502e416265f1fa78300347ad02a4e1f Mon Sep 17 00:00:00 2001 From: Attila Szabo Date: Wed, 15 Feb 2017 09:30:16 +0100 Subject: [PATCH] SQOOP-3061: Improve query validation in connection with --options-file thus correct options/queries like "--query SELECT * FROM test WHERE a = 'b'" will not fail with "Malformed option in options file" error message (Eric Lin via Attila Szabo) --- .../apache/sqoop/util/OptionsFileUtil.java | 24 ++- .../sqoop/util/TestOptionsFileExpansion.java | 156 ++++++++++++++++++ 2 files changed, 173 insertions(+), 7 deletions(-) diff --git a/src/java/org/apache/sqoop/util/OptionsFileUtil.java b/src/java/org/apache/sqoop/util/OptionsFileUtil.java index c476e005..993ac1b1 100644 --- a/src/java/org/apache/sqoop/util/OptionsFileUtil.java +++ b/src/java/org/apache/sqoop/util/OptionsFileUtil.java @@ -24,7 +24,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -147,10 +149,10 @@ private static String removeQuotesEncolosingOption( * Removes the surrounding quote characters from the given string. The quotes * are identified by the quote parameter, the given string by option. The * fileName parameter is used for raising exceptions with relevant message. - * @param fileName - * @param option - * @param quote - * @return + * @param fileName String The name of the file that contains sqoop options + * @param option String the actual options in the options file + * @param quote char The quote that we need to validate on, either single or double quotes + * @return String The validated options string * @throws Exception */ private static String removeQuoteCharactersIfNecessary(String fileName, @@ -167,11 +169,19 @@ private static String removeQuoteCharactersIfNecessary(String fileName, } if (startingQuote || endingQuote) { - throw new Exception("Malformed option in options file(" - + fileName + "): " + option); + // Regular expression looks like below: + // .*=\s*".*"$ OR .*=\s*'.*'$ + // it tries to match the following: + // .... column_name = "values" OR .... column_name = 'values' + // so that the query like: + // SELECT * FROM table WHERE column = "values" + // is valid even though it ends with double quote but no starting double quote + if (!Pattern.matches(".*=\\s*"+quote+".*"+quote+"$", option)) { + throw new Exception("Malformed option in options file(" + + fileName + "): " + option); + } } return option; } - } diff --git a/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java b/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java index 6d3f0f35..d403f3bb 100644 --- a/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java +++ b/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java @@ -165,6 +165,162 @@ public void testMultilineQuotedText() { } } + @Test + public void testValidFreeFormQueryNoQuotes() throws Exception { + String[] input = new String[]{ + "--query", + "SELECT * FROM table", + }; + + String[] output = new String[] { + "--query", + "SELECT * FROM table", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQuerySingleQuotesStartAndEnd() throws Exception { + String[] input = new String[]{ + "--query", + "'SELECT * FROM table'", + }; + + String[] output = new String[]{ + "--query", + "SELECT * FROM table", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQueryDoubleQuotesStartAndEnd() throws Exception { + String[] input = new String[]{ + "--query", + "\"SELECT * FROM table\"", + }; + + String[] output = new String[]{ + "--query", + "SELECT * FROM table", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQuerySingleQuotesInWhere() throws Exception { + String[] input = new String[]{ + "--query", + "SELECT * FROM table WHERE a = '1'", + }; + + String[] output = new String[]{ + "--query", + "SELECT * FROM table WHERE a = '1'", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQuerySingleAndDoubleQuotesInWhere() throws Exception { + String[] input = new String[] { + "--query", + "SELECT * FROM table WHERE a = '1' AND b = \"testing\"", + }; + + String[] output = new String[] { + "--query", + "SELECT * FROM table WHERE a = '1' AND b = \"testing\"", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQueryQuotesInTableNameAndColumnName() throws Exception { + String[] input = new String[] { + "--query", + "select * from `test\"test` where `c'c` = 'a'", + }; + + String[] output = new String[] { + "--query", + "select * from `test\"test` where `c'c` = 'a'", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQueryQuotesInTableNameAndColumnName2() throws Exception { + String[] input = new String[] { + "--query", + "select * from `test\"test` where `c'c` = 'a\"'", + }; + + String[] output = new String[] { + "--query", + "select * from `test\"test` where `c'c` = 'a\"'", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQueryQuotesInTableNameAndColumnName3() throws Exception { + String[] input = new String[] { + "--query", + "select * from `test\"test` where `c'c` = \"\"", + }; + + String[] output = new String[] { + "--query", + "select * from `test\"test` where `c'c` = \"\"", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQueryQuotesInTableNameAndColumnName4() throws Exception { + String[] input = new String[] { + "--query", + "select * from test where a = \"\\\"\"", + }; + + String[] output = new String[] { + "--query", + "select * from test where a = \"\\\"\"", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testInvalidFreeFormQueryEndingSingleQuoteOnly() throws Exception { + String[] input = new String[]{ + "--query", + "SELECT * FROM table'", + }; + + checkInvalidOptionsFile(input); + } + + @Test + public void testInvalidFreeFormQuerySingleQuoteStartDoubleQuoteEnd() throws Exception { + + String[] input = new String[]{ + "--query", + "'SELECT * FROM table\"", + }; + + checkInvalidOptionsFile(input); + } + private void checkInvalidOptionsFile(String[] fileContents) { try { checkOptionsFile(fileContents, new String[] {});