diff --git a/src/docs/user/tools.txt b/src/docs/user/tools.txt index 633d8129..a05dd8bf 100644 --- a/src/docs/user/tools.txt +++ b/src/docs/user/tools.txt @@ -161,6 +161,81 @@ arguments are not typically used with Sqoop, but they are included as part of Hadoop's internal argument-parsing system. +Using Options Files to Pass Arguments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When using Sqoop, the command line options that do not change from +invocation to invocation can be put in an options file for convenience. +An options file is a text file where each line identifies an option in +the order that it appears otherwise on the command line. Option files +allow specifying a single option on multiple lines by using the +back-slash character at the end of intermediate lines. Also supported +are comments within option files that begin with the hash character. +Comments must be specified on a new line and may not be mixed with +option text. All comments and empty lines are ignored when option +files are expanded. Unless options appear as quoted strings, any +leading or trailing spaces are ignored. Quoted strings if used must +not extend beyond the line on which they are specified. + +Option files can be specified anywhere in the command line as long as +the options within them follow the otherwise prescribed rules of +options ordering. For instance, regardless of where the options are +loaded from, they must follow the ordering such that generic options +appear first, tool specific options next, finally followed by options +that are intended to be passed to child programs. + +To specify an options file, simply create an options file in a +convenient location and pass it to the command line via ++\--options-file+ argument. + +Whenever an options file is specified, it is expanded on the +command line before the tool is invoked. You can specify more than +one option files within the same invocation if needed. + +For example, the following Sqoop invocation for import can +be specified alternatively as shown below: + +---- +$ sqoop import --connect jdbc:mysql://localhost/db --username foo --table TEST + +$ sqoop --options-file /users/homer/work/import.txt --table TEST +---- + +where the options file +/users/homer/work/import.txt+ contains the following: + +---- +import +--connect +jdbc:mysql://localhost/db +--username +foo +---- + +The options file can have empty lines and comments for readability purposes. +So the above example would work exactly the same if the options file ++/users/homer/work/import.txt+ contained the following: + +---- +# +# Options file for Sqoop import +# + +# Specifies the tool being invoked +import + +# Connect parameter and value +--connect +jdbc:mysql://localhost/db + +# Username parameter and value +--username +foo + +# +# Remaining options should be specified in the command line. +# +---- + Using Tools ~~~~~~~~~~~ diff --git a/src/java/com/cloudera/sqoop/Sqoop.java b/src/java/com/cloudera/sqoop/Sqoop.java index dcd0cfd1..90764404 100644 --- a/src/java/com/cloudera/sqoop/Sqoop.java +++ b/src/java/com/cloudera/sqoop/Sqoop.java @@ -29,6 +29,7 @@ import com.cloudera.sqoop.cli.ToolOptions; import com.cloudera.sqoop.tool.SqoopTool; +import com.cloudera.sqoop.util.OptionsFileUtil; /** * Main entry-point for Sqoop @@ -37,14 +38,21 @@ */ public class Sqoop extends Configured implements Tool { - public static final Log SQOOP_LOG = LogFactory.getLog("com.cloudera.sqoop"); + public static final Log SQOOP_LOG = LogFactory.getLog("com.cloudera.sqoop"); public static final Log LOG = LogFactory.getLog(Sqoop.class.getName()); - /** If this System property is set, always throw an exception, do not just - exit with status 1. - */ + /** + * If this System property is set, always throw an exception, do not just + * exit with status 1. + */ public static final String SQOOP_RETHROW_PROPERTY = "sqoop.throwOnError"; + /** + * The option to specify an options file from which other options to the + * tool are read. + */ + public static final String SQOOP_OPTIONS_FILE_SPECIFIER = "--options-file"; + static { Configuration.addDefaultResource("sqoop-site.xml"); } @@ -121,7 +129,7 @@ public int run(String [] args) { tool.appendArgs(this.childPrgmArgs); tool.validateOptions(options); } catch (Exception e) { - // Couldn't parse arguments. + // Couldn't parse arguments. // Log the stack trace for this exception LOG.debug(e.getMessage(), e); // Print exception message. @@ -144,7 +152,7 @@ public int run(String [] args) { * the child-program arguments in advance, and store them to be readded * later. * @param argv the argv in to the SqoopTool - * @return the argv with a "--" and any subsequent arguments removed. + * @return the argv with a "--" and any subsequent arguments removed. */ private String [] stashChildPrgmArgs(String [] argv) { for (int i = 0; i < argv.length; i++) { @@ -159,7 +167,7 @@ public int run(String [] args) { } /** - * Given a Sqoop object and a set of arguments to deliver to + * Given a Sqoop object and a set of arguments to deliver to * its embedded SqoopTool, run the tool, wrapping the call to * ToolRunner. * This entry-point is preferred to ToolRunner.run() because @@ -186,7 +194,18 @@ public static int runSqoop(Sqoop sqoop, String [] args) { * but does not call System.exit() as main() will. */ public static int runTool(String [] args) { - String toolName = args[0]; + // Expand the options + String[] expandedArgs = null; + try { + expandedArgs = OptionsFileUtil.expandArguments(args); + } catch (Exception ex) { + LOG.error("Error while expanding arguments", ex); + System.err.println(ex.getMessage()); + System.err.println("Try 'sqoop help' for usage."); + return 1; + } + + String toolName = expandedArgs[0]; SqoopTool tool = SqoopTool.getTool(toolName); if (null == tool) { System.err.println("No such sqoop tool: " + toolName @@ -194,8 +213,10 @@ public static int runTool(String [] args) { return 1; } + Sqoop sqoop = new Sqoop(tool); - return runSqoop(sqoop, Arrays.copyOfRange(args, 1, args.length)); + return runSqoop(sqoop, + Arrays.copyOfRange(expandedArgs, 1, expandedArgs.length)); } public static void main(String [] args) { diff --git a/src/java/com/cloudera/sqoop/util/OptionsFileUtil.java b/src/java/com/cloudera/sqoop/util/OptionsFileUtil.java new file mode 100644 index 00000000..af59896f --- /dev/null +++ b/src/java/com/cloudera/sqoop/util/OptionsFileUtil.java @@ -0,0 +1,179 @@ +/** + * Licensed to Cloudera, Inc. under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Cloudera, Inc. licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.cloudera.sqoop.util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.cloudera.sqoop.Sqoop; + +/** + * Provides utility functions to read in options file. An options file is a + * regular text file with each line specifying a separate option. An option + * may continue into a following line by using a back-slash separator character + * at the end of the non-terminating line. Options file also allow empty lines + * and comment lines which are disregarded. Comment lines must begin with the + * hash character as the first character. Leading and trailing white-spaces are + * ignored for any options read from the Options file. + */ +public final class OptionsFileUtil { + + public static final Log LOG = LogFactory.getLog( + OptionsFileUtil.class.getName()); + + /** + * Expands any options file that may be present in the given set of arguments. + * + * @param args the given arguments + * @return a new string array that contains the expanded arguments. + * @throws Exception + */ + public static String[] expandArguments(String[] args) throws Exception { + List options = new ArrayList(); + + for (int i = 0; i < args.length; i++) { + if (args[i].equals(Sqoop.SQOOP_OPTIONS_FILE_SPECIFIER)) { + if (i == args.length - 1) { + throw new Exception("Missing options file"); + } + + String fileName = args[++i]; + File optionsFile = new File(fileName); + BufferedReader reader = null; + StringBuilder buffer = new StringBuilder(); + try { + reader = new BufferedReader(new FileReader(optionsFile)); + String nextLine = null; + while ((nextLine = reader.readLine()) != null) { + nextLine = nextLine.trim(); + if (nextLine.length() == 0 || nextLine.startsWith("#")) { + // empty line or comment + continue; + } + + buffer.append(nextLine); + if (nextLine.endsWith("\\")) { + if (buffer.charAt(0) == '\'' || buffer.charAt(0) == '"') { + throw new Exception( + "Multiline quoted strings not supported in file(" + + fileName + "): " + buffer.toString()); + } + // Remove the trailing back-slash and continue + buffer.deleteCharAt(buffer.length() - 1); + } else { + // The buffer contains a full option + options.add( + removeQuotesEncolosingOption(fileName, buffer.toString())); + buffer.delete(0, buffer.length()); + } + } + + // Assert that the buffer is empty + if (buffer.length() != 0) { + throw new Exception("Malformed option in options file(" + + fileName + "): " + buffer.toString()); + } + } catch (IOException ex) { + throw new Exception("Unable to read options file: " + fileName, ex); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException ex) { + LOG.info("Exception while closing reader", ex); + } + } + } + } else { + // Regular option. Parse it and put it on the appropriate list + options.add(args[i]); + } + } + + return options.toArray(new String[options.size()]); + } + + /** + * Removes the surrounding quote characters as needed. It first attempts to + * remove surrounding double quotes. If successful, the resultant string is + * returned. If no surrounding double quotes are found, it attempts to remove + * surrounding single quote characters. If successful, the resultant string + * is returned. If not the original string is returnred. + * @param fileName + * @param option + * @return + * @throws Exception + */ + private static String removeQuotesEncolosingOption( + String fileName, String option) throws Exception { + + // Attempt to remove double quotes. If successful, return. + String option1 = removeQuoteCharactersIfNecessary(fileName, option, '"'); + if (!option1.equals(option)) { + // Quotes were successfully removed + return option1; + } + + // Attempt to remove single quotes. + return removeQuoteCharactersIfNecessary(fileName, option, '\''); + } + + /** + * Removes the surrounding quote characters from the given string. The quotes + * are identified by the quote parameter, the given string by option. The + * fileName parameter is used for raising exceptions with relevant message. + * @param fileName + * @param option + * @param quote + * @return + * @throws Exception + */ + private static String removeQuoteCharactersIfNecessary(String fileName, + String option, char quote) throws Exception { + boolean startingQuote = (option.charAt(0) == quote); + boolean endingQuote = (option.charAt(option.length() - 1) == quote); + + if (startingQuote && endingQuote) { + if (option.length() == 1) { + throw new Exception("Malformed option in options file(" + + fileName + "): " + option); + } + return option.substring(1, option.length() - 1); + } + + if (startingQuote || endingQuote) { + throw new Exception("Malformed option in options file(" + + fileName + "): " + option); + } + + return option; + } + + private OptionsFileUtil() { + // Disable object creation + } + +} diff --git a/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java b/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java new file mode 100644 index 00000000..6c214632 --- /dev/null +++ b/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java @@ -0,0 +1,278 @@ +/** + * Licensed to Cloudera, Inc. under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Cloudera, Inc. licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.cloudera.sqoop.util; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; + +import junit.framework.TestCase; + +import org.junit.Assert; + +import com.cloudera.sqoop.Sqoop; + +/** + * Tests various options file loading scenarios. + */ +public class TestOptionsFileExpansion extends TestCase { + + /** + * Text from options file 1. Each string represents a new line. + */ + private static final String[] OPTIONS_FILE_TEXT1 = new String[] { + "--foo", + "-bar", + "--", + "--XYZ", + }; + + /** + * Expected options parsed out from options file 1. + */ + private static final String[] OPTIONS_FILE_TEXT1_OUTPUT = new String[] { + "--foo", + "-bar", + "--", + "--XYZ", + }; + + /** + * Text for options file 2. Each string represents a new line. This + * contains empty lines, comments and optinos that extend to multiple lines. + */ + private static final String[] OPTIONS_FILE_TEXT2 = new String[] { + "--archives", + "tools.jar,archive.jar,test.jar,\\", + "ldap.jar,sasl.jar", + "--connect", + "jdbc:jdbcspy:localhost:1521:test", + "--username", + "superman", + "--password", + "", + "# Ironic password.", + "# No one will ever guess.", + "kryptonite", + }; + + /** + * Expected options parsed out from file 2. + */ + private static final String[] OPTIONS_FILE_TEXT2_OUTPUT = new String[] { + "--archives", + "tools.jar,archive.jar,test.jar,ldap.jar,sasl.jar", + "--connect", + "jdbc:jdbcspy:localhost:1521:test", + "--username", + "superman", + "--password", + "kryptonite", + }; + + /** + * Text for options file 4. This contains options that represent empty + * strings or strings that have leading and trailing spaces. + */ + private static final String[] OPTIONS_FILE_TEXT3 = new String[] { + "-", + "\" leading spaces\"", + "' leading and trailing spaces '", + "\"\"", + "''", + }; + + /** + * Expected options parsed out from file 3. + */ + private static final String[] OPTIONS_FILE_TEXT3_OUTPUT = new String[] { + "-", + " leading spaces", + " leading and trailing spaces ", + "", + "", + }; + + /** + * Text for options file 4. This file has an invalid entry in the last line + * which will cause it to fail to load. + */ + private static final String[] OPTIONS_FILE_TEXT4 = new String[] { + "--abcd", + "--efgh", + "# foo", + "# bar", + "XYZ\\", + }; + + /** + * Text for options file 5. This file has an invalid entry in the second line + * where there is a starting single quote character that is not terminating. + */ + private static final String[] OPTIONS_FILE_TEXT5 = new String[] { + "-abcd", + "\'", + "--foo", + }; + + /** + * Text for options file 6. This file has an invalid entry in the second line + * where a quoted string extends into the following line. + */ + private static final String[] OPTIONS_FILE_TEXT6 = new String[] { + "--abcd", + "' the quick brown fox \\", + "jumped over the lazy dog'", + "--efgh", + }; + + public void testOptionsFiles() throws Exception { + checkOptionsFile(OPTIONS_FILE_TEXT1, OPTIONS_FILE_TEXT1_OUTPUT); + checkOptionsFile(OPTIONS_FILE_TEXT2, OPTIONS_FILE_TEXT2_OUTPUT); + checkOptionsFile(OPTIONS_FILE_TEXT3, OPTIONS_FILE_TEXT3_OUTPUT); + } + + public void testInvalidOptionsFile() { + checkInvalidOptionsFile(OPTIONS_FILE_TEXT4); + checkInvalidOptionsFile(OPTIONS_FILE_TEXT5); + } + + public void testMultilineQuotedText() { + try { + checkOptionsFile(OPTIONS_FILE_TEXT6, new String[] {}); + Assert.assertTrue(false); + } catch (Exception ex) { + Assert.assertTrue( + ex.getMessage().startsWith("Multiline quoted strings not supported")); + } + } + + private void checkInvalidOptionsFile(String[] fileContents) { + try { + checkOptionsFile(fileContents, new String[] {}); + Assert.assertTrue(false); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().startsWith("Malformed option")); + } + } + + private void checkOptionsFile(String[] fileContent, String[] expectedOptions) + throws Exception { + String[] prefix0 = new String[] { }; + String[] suffix0 = new String[] { }; + + checkOutput(prefix0, suffix0, fileContent, expectedOptions); + + String[] prefix1 = new String[] { "--nomnom" }; + String[] suffix1 = new String[] { }; + + checkOutput(prefix1, suffix1, + fileContent, expectedOptions); + + String[] prefix2 = new String[] { }; + String[] suffix2 = new String[] { "yIkes" }; + + checkOutput(prefix2, suffix2, + fileContent, expectedOptions); + + String[] prefix3 = new String[] { "foo", "bar" }; + String[] suffix3 = new String[] { "xyz", "abc" }; + + checkOutput(prefix3, suffix3, + fileContent, expectedOptions); + } + + + /** + * Uses the given prefix and suffix to create the original args array which + * contains two entries between the prefix and suffix entries that specify + * the options file. The options file is dynamically created using the + * contents of the third array - fileContent. Once this is expanded, the + * expanded arguments are compared to see if they are same as prefix entries + * followed by parsed arguments from the options file, followed by suffix + * entries. + * @param prefix + * @param suffix + * @param fileContent + * @param expectedContent + * @throws Exception + */ + private void checkOutput(String[] prefix, String[] suffix, + String[] fileContent, String[] expectedContent) throws Exception { + + String[] args = new String[prefix.length + 2 + suffix.length]; + + for (int i = 0; i < prefix.length; i++) { + args[i] = prefix[i]; + } + args[prefix.length] = Sqoop.SQOOP_OPTIONS_FILE_SPECIFIER; + args[prefix.length + 1] = createOptionsFile(fileContent); + for (int j = 0; j < suffix.length; j++) { + args[j + 2 + prefix.length] = suffix[j]; + } + + String[] expandedArgs = OptionsFileUtil.expandArguments(args); + + assertSame(prefix, expectedContent, suffix, expandedArgs); + + } + + private void assertSame(String[] prefix, String[] content, String[] suffix, + String[] actual) { + Assert.assertTrue(prefix.length + content.length + suffix.length + == actual.length); + + for (int i = 0; i < prefix.length; i++) { + Assert.assertTrue(actual[i].equals(prefix[i])); + } + + for (int i = 0; i < content.length; i++) { + Assert.assertTrue(actual[i + prefix.length].equals(content[i])); + } + + for (int i = 0; i < suffix.length; i++) { + Assert.assertTrue(actual[i + prefix.length + content.length].equals( + suffix[i])); + } + } + + private String createOptionsFile(String[] data) throws Exception { + File file = File.createTempFile("options", ".opf"); + file.deleteOnExit(); + + BufferedWriter writer = null; + try { + writer = new BufferedWriter(new FileWriter(file)); + for (String datum : data) { + writer.write(datum); + writer.newLine(); + } + } finally { + if (writer != null) { + try { + writer.close(); + } catch (IOException ex) { + // No handling required + } + } + } + + return file.getAbsolutePath(); + } +}