SQOOP-126. Support for loading options from file.

This change allows Sqoop to load options from an options file. An
options file is specified using --options-file. All options that
are otherwise specified on the command line should be specified
in this file in the order they would otherwise appear on the command
line. Options files can contain empty lines and comments for
readability. More than one options file may be used for a single
tool invocation if so preferred. Leading and trailing spaces are
ignored unless they appear within single or double quotes. Quoted
options extending into multiple lines are not supported.

From: Arvind Prabhakar <arvind@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149999 13f79535-47bb-0310-9956-ffa450edef68

src/docs/user/tools.txt

@@ -161,6 +161,81 @@ arguments are not typically used with Sqoop, but they are included as
 part of Hadoop's internal argument-parsing system.
 
 
+Using Options Files to Pass Arguments
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When using Sqoop, the command line options that do not change from
+invocation to invocation can be put in an options file for convenience.
+An options file is a text file where each line identifies an option in
+the order that it appears otherwise on the command line. Option files
+allow specifying a single option on multiple lines by using the
+back-slash character at the end of intermediate lines. Also supported
+are comments within option files that begin with the hash character.
+Comments must be specified on a new line and may not be mixed with
+option text. All comments and empty lines are ignored when option
+files are expanded. Unless options appear as quoted strings, any
+leading or trailing spaces are ignored. Quoted strings if used must
+not extend beyond the line on which they are specified.
+
+Option files can be specified anywhere in the command line as long as
+the options within them follow the otherwise prescribed rules of
+options ordering. For instance, regardless of where the options are
+loaded from, they must follow the ordering such that generic options
+appear first, tool specific options next, finally followed by options
+that are intended to be passed to child programs.
+
+To specify an options file, simply create an options file in a
+convenient location and pass it to the command line via
++\--options-file+ argument.
+
+Whenever an options file is specified, it is expanded on the
+command line before the tool is invoked. You can specify more than
+one option files within the same invocation if needed.
+
+For example, the following Sqoop invocation for import can
+be specified alternatively as shown below:
+
+----
+$ sqoop import --connect jdbc:mysql://localhost/db --username foo --table TEST
+
+$ sqoop --options-file /users/homer/work/import.txt --table TEST
+----
+
+where the options file +/users/homer/work/import.txt+ contains the following:
+
+----
+import
+--connect
+jdbc:mysql://localhost/db
+--username
+foo
+----
+
+The options file can have empty lines and comments for readability purposes.
+So the above example would work exactly the same if the options file
++/users/homer/work/import.txt+ contained the following:
+
+----
+#
+# Options file for Sqoop import
+#
+
+# Specifies the tool being invoked
+import
+
+# Connect parameter and value
+--connect
+jdbc:mysql://localhost/db
+
+# Username parameter and value
+--username
+foo
+
+#
+# Remaining options should be specified in the command line.
+#
+----
+
 Using Tools
 ~~~~~~~~~~~
 

src/java/com/cloudera/sqoop/Sqoop.java

@@ -29,6 +29,7 @@
 
 import com.cloudera.sqoop.cli.ToolOptions;
 import com.cloudera.sqoop.tool.SqoopTool;
+import com.cloudera.sqoop.util.OptionsFileUtil;
 
 /**
  * Main entry-point for Sqoop
@@ -40,11 +41,18 @@ public class Sqoop extends Configured implements Tool {
   public static final Log SQOOP_LOG = LogFactory.getLog("com.cloudera.sqoop");
   public static final Log LOG = LogFactory.getLog(Sqoop.class.getName());
 
-  /** If this System property is set, always throw an exception, do not just
-      exit with status 1.
+  /**
+   * If this System property is set, always throw an exception, do not just
+   * exit with status 1.
    */
   public static final String SQOOP_RETHROW_PROPERTY = "sqoop.throwOnError";
 
+  /**
+   * The option to specify an options file from which other options to the
+   * tool are read.
+   */
+  public static final String SQOOP_OPTIONS_FILE_SPECIFIER = "--options-file";
+
   static {
     Configuration.addDefaultResource("sqoop-site.xml");
   }
@@ -186,7 +194,18 @@ public static int runSqoop(Sqoop sqoop, String [] args) {
    * but does not call System.exit() as main() will.
    */
   public static int runTool(String [] args) {
-    String toolName = args[0];
+    // Expand the options
+    String[] expandedArgs = null;
+    try {
+      expandedArgs = OptionsFileUtil.expandArguments(args);
+    } catch (Exception ex) {
+      LOG.error("Error while expanding arguments", ex);
+      System.err.println(ex.getMessage());
+      System.err.println("Try 'sqoop help' for usage.");
+      return 1;
+    }
+
+    String toolName = expandedArgs[0];
     SqoopTool tool = SqoopTool.getTool(toolName);
     if (null == tool) {
       System.err.println("No such sqoop tool: " + toolName
@@ -194,8 +213,10 @@ public static int runTool(String [] args) {
       return 1;
     }
 
+
     Sqoop sqoop = new Sqoop(tool);
-    return runSqoop(sqoop, Arrays.copyOfRange(args, 1, args.length));
+    return runSqoop(sqoop,
+        Arrays.copyOfRange(expandedArgs, 1, expandedArgs.length));
   }
 
   public static void main(String [] args) {

src/java/com/cloudera/sqoop/util/OptionsFileUtil.java

@@ -0,0 +1,179 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.sqoop.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.cloudera.sqoop.Sqoop;
+
+/**
+ * Provides utility functions to read in options file. An options file is a
+ * regular text file with each line specifying a separate option. An option
+ * may continue into a following line by using a back-slash separator character
+ * at the end of the non-terminating line. Options file also allow empty lines
+ * and comment lines which are disregarded. Comment lines must begin with the
+ * hash character as the first character. Leading and trailing white-spaces are
+ * ignored for any options read from the Options file.
+ */
+public final class OptionsFileUtil {
+
+  public static final Log LOG = LogFactory.getLog(
+                                  OptionsFileUtil.class.getName());
+
+  /**
+   * Expands any options file that may be present in the given set of arguments.
+   *
+   * @param args the given arguments
+   * @return a new string array that contains the expanded arguments.
+   * @throws Exception
+   */
+  public static String[] expandArguments(String[] args) throws Exception {
+    List<String> options = new ArrayList<String>();
+
+    for (int i = 0; i < args.length; i++) {
+      if (args[i].equals(Sqoop.SQOOP_OPTIONS_FILE_SPECIFIER)) {
+        if (i == args.length - 1) {
+          throw new Exception("Missing options file");
+        }
+
+        String fileName = args[++i];
+        File optionsFile = new File(fileName);
+        BufferedReader reader = null;
+        StringBuilder buffer = new StringBuilder();
+        try {
+          reader = new BufferedReader(new FileReader(optionsFile));
+          String nextLine = null;
+          while ((nextLine = reader.readLine()) != null) {
+            nextLine = nextLine.trim();
+            if (nextLine.length() == 0 || nextLine.startsWith("#")) {
+              // empty line or comment
+              continue;
+            }
+
+            buffer.append(nextLine);
+            if (nextLine.endsWith("\\")) {
+              if (buffer.charAt(0) == '\'' || buffer.charAt(0) == '"') {
+                throw new Exception(
+                    "Multiline quoted strings not supported in file("
+                      + fileName + "): " + buffer.toString());
+              }
+              // Remove the trailing back-slash and continue
+              buffer.deleteCharAt(buffer.length()  - 1);
+            } else {
+              // The buffer contains a full option
+              options.add(
+                  removeQuotesEncolosingOption(fileName, buffer.toString()));
+              buffer.delete(0, buffer.length());
+            }
+          }
+
+          // Assert that the buffer is empty
+          if (buffer.length() != 0) {
+            throw new Exception("Malformed option in options file("
+                + fileName + "): " + buffer.toString());
+          }
+        } catch (IOException ex) {
+          throw new Exception("Unable to read options file: " + fileName, ex);
+        } finally {
+          if (reader != null) {
+            try {
+              reader.close();
+            } catch (IOException ex) {
+              LOG.info("Exception while closing reader", ex);
+            }
+          }
+        }
+      } else {
+        // Regular option. Parse it and put it on the appropriate list
+        options.add(args[i]);
+      }
+    }
+
+    return options.toArray(new String[options.size()]);
+  }
+
+  /**
+   * Removes the surrounding quote characters as needed. It first attempts to
+   * remove surrounding double quotes. If successful, the resultant string is
+   * returned. If no surrounding double quotes are found, it attempts to remove
+   * surrounding single quote characters. If successful, the resultant string
+   * is returned. If not the original string is returnred.
+   * @param fileName
+   * @param option
+   * @return
+   * @throws Exception
+   */
+  private static String removeQuotesEncolosingOption(
+      String fileName, String option) throws Exception {
+
+    // Attempt to remove double quotes. If successful, return.
+    String option1 = removeQuoteCharactersIfNecessary(fileName, option, '"');
+    if (!option1.equals(option)) {
+      // Quotes were successfully removed
+      return option1;
+    }
+
+    // Attempt to remove single quotes.
+    return removeQuoteCharactersIfNecessary(fileName, option, '\'');
+  }
+
+  /**
+   * Removes the surrounding quote characters from the given string. The quotes
+   * are identified by the quote parameter, the given string by option. The
+   * fileName parameter is used for raising exceptions with relevant message.
+   * @param fileName
+   * @param option
+   * @param quote
+   * @return
+   * @throws Exception
+   */
+  private static String removeQuoteCharactersIfNecessary(String fileName,
+      String option, char quote) throws Exception {
+    boolean startingQuote = (option.charAt(0) == quote);
+    boolean endingQuote = (option.charAt(option.length() - 1) == quote);
+
+    if (startingQuote && endingQuote) {
+      if (option.length() == 1) {
+        throw new Exception("Malformed option in options file("
+            + fileName + "): " + option);
+      }
+      return option.substring(1, option.length() - 1);
+    }
+
+    if (startingQuote || endingQuote) {
+       throw new Exception("Malformed option in options file("
+           + fileName + "): " + option);
+    }
+
+    return option;
+  }
+
+  private OptionsFileUtil() {
+    // Disable object creation
+  }
+
+}

src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java

@@ -0,0 +1,278 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.cloudera.sqoop.util;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.junit.Assert;
+
+import com.cloudera.sqoop.Sqoop;
+
+/**
+ * Tests various options file loading scenarios.
+ */
+public class TestOptionsFileExpansion extends TestCase {
+
+  /**
+   * Text from options file 1. Each string represents a new line.
+   */
+  private static final String[] OPTIONS_FILE_TEXT1 = new String[] {
+    "--foo",
+    "-bar",
+    "--",
+    "--XYZ",
+  };
+
+  /**
+   * Expected options parsed out from options file 1.
+   */
+  private static final String[] OPTIONS_FILE_TEXT1_OUTPUT = new String[] {
+    "--foo",
+    "-bar",
+    "--",
+    "--XYZ",
+  };
+
+  /**
+   * Text for options file 2. Each string represents a new line. This
+   * contains empty lines, comments and optinos that extend to multiple lines.
+   */
+  private static final String[] OPTIONS_FILE_TEXT2 = new String[] {
+    "--archives",
+    "tools.jar,archive.jar,test.jar,\\",
+    "ldap.jar,sasl.jar",
+    "--connect",
+    "jdbc:jdbcspy:localhost:1521:test",
+    "--username",
+    "superman",
+    "--password",
+    "",
+    "# Ironic password.",
+    "# No one will ever guess.",
+    "kryptonite",
+  };
+
+  /**
+   * Expected options parsed out from file 2.
+   */
+  private static final String[] OPTIONS_FILE_TEXT2_OUTPUT = new String[] {
+    "--archives",
+    "tools.jar,archive.jar,test.jar,ldap.jar,sasl.jar",
+    "--connect",
+    "jdbc:jdbcspy:localhost:1521:test",
+    "--username",
+    "superman",
+    "--password",
+    "kryptonite",
+  };
+
+  /**
+   * Text for options file 4. This contains options that represent empty
+   * strings or strings that have leading and trailing spaces.
+   */
+  private static final String[] OPTIONS_FILE_TEXT3 = new String[] {
+    "-",
+    "\"     leading spaces\"",
+    "'        leading and trailing spaces    '",
+    "\"\"",
+    "''",
+  };
+
+  /**
+   * Expected options parsed out from file 3.
+   */
+  private static final String[] OPTIONS_FILE_TEXT3_OUTPUT = new String[] {
+    "-",
+    "     leading spaces",
+    "        leading and trailing spaces    ",
+    "",
+    "",
+  };
+
+  /**
+   * Text for options file 4. This file has an invalid entry in the last line
+   * which will cause it to fail to load.
+   */
+  private static final String[] OPTIONS_FILE_TEXT4 = new String[] {
+    "--abcd",
+    "--efgh",
+    "# foo",
+    "# bar",
+    "XYZ\\",
+  };
+
+  /**
+   * Text for options file 5. This file has an invalid entry in the second line
+   * where there is a starting single quote character that is not terminating.
+   */
+  private static final String[] OPTIONS_FILE_TEXT5 = new String[] {
+    "-abcd",
+    "\'",
+    "--foo",
+  };
+
+  /**
+   * Text for options file 6. This file has an invalid entry in the second line
+   * where a quoted string extends into the following line.
+   */
+  private static final String[] OPTIONS_FILE_TEXT6 = new String[] {
+    "--abcd",
+    "' the quick brown fox \\",
+    "jumped over the lazy dog'",
+    "--efgh",
+  };
+
+  public void testOptionsFiles() throws Exception {
+    checkOptionsFile(OPTIONS_FILE_TEXT1, OPTIONS_FILE_TEXT1_OUTPUT);
+    checkOptionsFile(OPTIONS_FILE_TEXT2, OPTIONS_FILE_TEXT2_OUTPUT);
+    checkOptionsFile(OPTIONS_FILE_TEXT3, OPTIONS_FILE_TEXT3_OUTPUT);
+  }
+
+  public void testInvalidOptionsFile() {
+    checkInvalidOptionsFile(OPTIONS_FILE_TEXT4);
+    checkInvalidOptionsFile(OPTIONS_FILE_TEXT5);
+  }
+
+  public void testMultilineQuotedText() {
+    try {
+      checkOptionsFile(OPTIONS_FILE_TEXT6, new String[] {});
+      Assert.assertTrue(false);
+    } catch (Exception ex) {
+      Assert.assertTrue(
+          ex.getMessage().startsWith("Multiline quoted strings not supported"));
+    }
+  }
+
+  private void checkInvalidOptionsFile(String[] fileContents) {
+    try {
+      checkOptionsFile(fileContents, new String[] {});
+      Assert.assertTrue(false);
+    } catch (Exception ex) {
+      Assert.assertTrue(ex.getMessage().startsWith("Malformed option"));
+    }
+  }
+
+  private void checkOptionsFile(String[] fileContent, String[] expectedOptions)
+    throws Exception {
+    String[] prefix0 = new String[] { };
+    String[] suffix0 = new String[] { };
+
+    checkOutput(prefix0, suffix0, fileContent, expectedOptions);
+
+    String[] prefix1 = new String[] { "--nomnom" };
+    String[] suffix1 = new String[] { };
+
+    checkOutput(prefix1, suffix1,
+        fileContent, expectedOptions);
+
+    String[] prefix2 = new String[] { };
+    String[] suffix2 = new String[] { "yIkes" };
+
+    checkOutput(prefix2, suffix2,
+        fileContent, expectedOptions);
+
+    String[] prefix3 = new String[] { "foo", "bar" };
+    String[] suffix3 = new String[] { "xyz", "abc" };
+
+    checkOutput(prefix3, suffix3,
+        fileContent, expectedOptions);
+  }
+
+
+  /**
+   * Uses the given prefix and suffix to create the original args array which
+   * contains two entries between the prefix and suffix entries that specify
+   * the options file. The options file is dynamically created using the
+   * contents of the third array - fileContent. Once this is expanded, the
+   * expanded arguments are compared to see if they are same as prefix entries
+   * followed by parsed arguments from the options file, followed by suffix
+   * entries.
+   * @param prefix
+   * @param suffix
+   * @param fileContent
+   * @param expectedContent
+   * @throws Exception
+   */
+  private void checkOutput(String[] prefix, String[] suffix,
+      String[] fileContent, String[] expectedContent) throws Exception {
+
+    String[] args = new String[prefix.length + 2 + suffix.length];
+
+    for (int i = 0; i < prefix.length; i++) {
+      args[i] = prefix[i];
+    }
+    args[prefix.length] = Sqoop.SQOOP_OPTIONS_FILE_SPECIFIER;
+    args[prefix.length + 1] =  createOptionsFile(fileContent);
+    for (int j = 0; j < suffix.length; j++) {
+      args[j + 2 + prefix.length] = suffix[j];
+    }
+
+    String[] expandedArgs = OptionsFileUtil.expandArguments(args);
+
+    assertSame(prefix, expectedContent, suffix, expandedArgs);
+
+  }
+
+  private void assertSame(String[] prefix, String[] content, String[] suffix,
+      String[] actual) {
+    Assert.assertTrue(prefix.length + content.length + suffix.length
+        == actual.length);
+
+    for (int i = 0; i < prefix.length; i++) {
+      Assert.assertTrue(actual[i].equals(prefix[i]));
+    }
+
+    for (int i = 0; i < content.length; i++) {
+      Assert.assertTrue(actual[i + prefix.length].equals(content[i]));
+    }
+
+    for (int i = 0; i < suffix.length; i++) {
+      Assert.assertTrue(actual[i + prefix.length + content.length].equals(
+          suffix[i]));
+    }
+  }
+
+  private String createOptionsFile(String[] data) throws Exception {
+    File file = File.createTempFile("options", ".opf");
+    file.deleteOnExit();
+
+    BufferedWriter writer = null;
+    try {
+      writer = new BufferedWriter(new FileWriter(file));
+      for (String datum : data) {
+        writer.write(datum);
+        writer.newLine();
+      }
+    } finally {
+      if (writer != null) {
+        try {
+          writer.close();
+        } catch (IOException ex) {
+          // No handling required
+        }
+      }
+    }
+
+    return file.getAbsolutePath();
+  }
+}