diff --git a/src/java/com/cloudera/sqoop/cli/RelatedOptions.java b/src/java/com/cloudera/sqoop/cli/RelatedOptions.java index 41ce6ec8..d0bd584a 100644 --- a/src/java/com/cloudera/sqoop/cli/RelatedOptions.java +++ b/src/java/com/cloudera/sqoop/cli/RelatedOptions.java @@ -1,6 +1,4 @@ /** - * Copyright 2011 The Apache Software Foundation - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,35 +15,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.cloudera.sqoop.cli; -import org.apache.commons.cli.Options; - /** - * Class that extends a set of options to provide a title for a - * related set of options. + * @deprecated Moving to use org.apache.sqoop namespace. */ -public class RelatedOptions extends Options { - - private String relatedTitle; +public class RelatedOptions + extends org.apache.sqoop.cli.RelatedOptions { public RelatedOptions() { - this(""); + super(); } public RelatedOptions(final String title) { - super(); - this.relatedTitle = title; - } - - public String getTitle() { - return relatedTitle; - } - - @Override - public String toString() { - return relatedTitle + "\n" + super.toString(); + super(title); } } diff --git a/src/java/com/cloudera/sqoop/cli/SqoopParser.java b/src/java/com/cloudera/sqoop/cli/SqoopParser.java index 9e36bbf5..0c6d84e0 100644 --- a/src/java/com/cloudera/sqoop/cli/SqoopParser.java +++ b/src/java/com/cloudera/sqoop/cli/SqoopParser.java @@ -1,6 +1,4 @@ /** - * Copyright 2011 The Apache Software Foundation - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,104 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.cloudera.sqoop.cli; -import java.util.ListIterator; - -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.MissingArgumentException; -import org.apache.commons.cli.ParseException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.StringUtils; - /** - * Options parser that follows Sqoop's specific options-parsing needs. - * + * @deprecated Moving to use org.apache.sqoop namespace. */ -public class SqoopParser extends GnuParser { - - public static final Log LOG = LogFactory.getLog(SqoopParser.class.getName()); - - // We need a handle to Option.addValueForProcessing(String). - // Since Hadoop will load this in a different classloader than - // this Sqoop class, we cannot see their package-specific methods. - // So we just call it by reflection. As long as we're at it, this - // allows us to also put SqoopParser in its own package. - private static java.lang.reflect.Method addValForProcessing; - - static { - try { - addValForProcessing = Option.class.getDeclaredMethod( - "addValueForProcessing", String.class); - addValForProcessing.setAccessible(true); - } catch (NoSuchMethodException nsme) { - LOG.error("Could not load required method of Parser: " - + StringUtils.stringifyException(nsme)); - addValForProcessing = null; - } - } - - @Override - /** - * Processes arguments to options but only strips matched quotes. - */ - public void processArgs(Option opt, ListIterator iter) - throws ParseException { - // Loop until an option is found. - while (iter.hasNext()) { - String str = (String) iter.next(); - - if (getOptions().hasOption(str) && str.startsWith("-")) { - // found an Option, not an argument. - iter.previous(); - break; - } - - // Otherwise, this is a value. - try { - // Note that we only strip matched quotes here. - addValForProcessing.invoke(opt, stripMatchedQuotes(str)); - } catch (IllegalAccessException iae) { - throw new RuntimeException(iae); - } catch (java.lang.reflect.InvocationTargetException ite) { - // Any runtime exception thrown within addValForProcessing() - // will be wrapped in an InvocationTargetException. - iter.previous(); - break; - } catch (RuntimeException re) { - iter.previous(); - break; - } - } - - if (opt.getValues() == null && !opt.hasOptionalArg()) { - throw new MissingArgumentException(opt); - } - } - - /** - * Util.stripLeadingAndTrailingQuotes() will strip a '"' - * character from either or both sides of a string. We only - * strip the matched pair. - */ - private String stripMatchedQuotes(String in) { - if (null == in || "\"".equals(in)) { - return in; // single quote stays as-is. - } else if(in.startsWith("\"") && in.endsWith("\"")) { - // Strip this pair of matched quotes. - return in.substring(1, in.length() - 1); - } else { - // return as-is. - return in; - } - } +public class SqoopParser + extends org.apache.sqoop.cli.SqoopParser { } diff --git a/src/java/com/cloudera/sqoop/cli/ToolOptions.java b/src/java/com/cloudera/sqoop/cli/ToolOptions.java index 2a18bed9..0b5da6c3 100644 --- a/src/java/com/cloudera/sqoop/cli/ToolOptions.java +++ b/src/java/com/cloudera/sqoop/cli/ToolOptions.java @@ -1,6 +1,4 @@ /** - * Copyright 2011 The Apache Software Foundation - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,134 +18,10 @@ package com.cloudera.sqoop.cli; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; - /** - * Class that holds several sets of related options, providing a container - * for all the options associated with a single tool. - * The order in which sets of related options are added to this tool is - * preserved in printing and iteration. + * @deprecated Moving to use org.apache.sqoop namespace. */ -public class ToolOptions implements Iterable { - - private List optGroups; - - public ToolOptions() { - this.optGroups = new ArrayList(); - } - - /** - * Add a block of related options to the options for this tool. - * @param opts the set of RelatedOptions to add. - */ - public void addOptions(RelatedOptions opts) { - optGroups.add(opts); - } - - /** - * Add a block of related options to the options for this tool, - * if a block has not already been added with the same title. - * @param opts the set of RelatedOptions to add. - */ - public void addUniqueOptions(RelatedOptions opts) { - if (!containsGroup(opts.getTitle())) { - optGroups.add(opts); - } - } - - /** - * Reports whether this collection of RelatedOptions contains - * a RelatedOptions with the specified title. - * @param title the group title to search for - * @return true if a RelatedOptions with this group title is - * in the collection. - */ - public boolean containsGroup(String title) { - for (RelatedOptions related : this) { - if (related.getTitle().equals(title)) { - return true; - } - } - - return false; - } - - /** - * Provide an iterator over all sets of RelatedOptions. - * @return an iterator returning each RelatedOptions element. - */ - public Iterator iterator() { - return optGroups.iterator(); - } - - - /** - * Flatten the different sets of related options into a single collection - * of options. - * @return all options in the ToolOptions as a single set - */ - public Options merge() { - Options mergedOpts = new Options(); - int totalOpts = 0; - for (RelatedOptions relatedOpts : this) { - for (Object optObj : relatedOpts.getOptions()) { - Option opt = (Option) optObj; - mergedOpts.addOption(opt); - totalOpts++; - } - } - - return mergedOpts; - } - - /** - * Print the help to the console using a default help formatter. - */ - public void printHelp() { - printHelp(new HelpFormatter()); - } - - /** - * Print the help to the console using the specified help formatter. - * @param formatter the HelpFormatter to use. - */ - public void printHelp(HelpFormatter formatter) { - printHelp(formatter, new PrintWriter(System.out, true)); - } - - /** - * Print the help to the specified PrintWriter, using the specified - * help formatter. - * @param formatter the HelpFormatter to use. - * @param pw the PrintWriter to emit to. - */ - public void printHelp(HelpFormatter formatter, PrintWriter pw) { - boolean first = true; - for (RelatedOptions optGroup : optGroups) { - if (!first) { - pw.println(""); - } - pw.println(optGroup.getTitle() + ":"); - formatter.printOptions(pw, formatter.getWidth(), optGroup, 0, 4); - first = false; - } - } - - @Override - public String toString() { - StringWriter sw = new StringWriter(); - printHelp(new HelpFormatter(), new PrintWriter(sw)); - sw.flush(); - return sw.getBuffer().toString(); - } - +public class ToolOptions + extends org.apache.sqoop.cli.ToolOptions { } diff --git a/src/java/com/cloudera/sqoop/config/ConfigurationConstants.java b/src/java/com/cloudera/sqoop/config/ConfigurationConstants.java index 01ab7e8a..72e7df13 100644 --- a/src/java/com/cloudera/sqoop/config/ConfigurationConstants.java +++ b/src/java/com/cloudera/sqoop/config/ConfigurationConstants.java @@ -1,6 +1,4 @@ /** - * Copyright 2011 The Apache Software Foundation - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,70 +15,37 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - package com.cloudera.sqoop.config; /** - * Static constants that identify configuration keys, counter group names, and - * counter names. + * @deprecated Moving to use org.apache.sqoop namespace. */ public final class ConfigurationConstants { - /** - * The Configuration property identifying the current task id. - */ - public static final String PROP_MAPRED_TASK_ID = "mapred.task.id"; - - /** - * The Configuration property identifying the job's local directory. - */ - public static final String PROP_JOB_LOCAL_DIRECTORY = "job.local.dir"; - - /** - * The Configuration property identifying the number of map tasks to be used. - */ - public static final String PROP_MAPRED_MAP_TASKS = "mapred.map.tasks"; - - /** - * The Configuration property identifying the speculative execution flag for - * map tasks. - */ + public static final String PROP_MAPRED_TASK_ID = + org.apache.sqoop.config.ConfigurationConstants.PROP_MAPRED_TASK_ID; + public static final String PROP_JOB_LOCAL_DIRECTORY = + org.apache.sqoop.config.ConfigurationConstants.PROP_JOB_LOCAL_DIRECTORY; + public static final String PROP_MAPRED_MAP_TASKS = + org.apache.sqoop.config.ConfigurationConstants.PROP_MAPRED_MAP_TASKS; public static final String PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC = - "mapred.map.tasks.speculative.execution"; - - /** - * The Configuration property identifying the speculative execution flag for - * reduce tasks. - */ + org.apache.sqoop.config. + ConfigurationConstants.PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC; public static final String PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC = - "mapred.reduce.tasks.speculative.execution"; - - /** - * The Configuration property identifying the job tracker address. - */ + org.apache.sqoop.config. + ConfigurationConstants.PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC; public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS = - "mapred.job.tracker"; - - /** - * The group name of task counters. - */ + org.apache.sqoop.config. + ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS; public static final String COUNTER_GROUP_MAPRED_TASK_COUNTERS = - "org.apache.hadoop.mapred.Task$Counter"; - - /** - * The name of the counter that tracks output records from Map phase. - */ + org.apache.sqoop.config. + ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS; public static final String COUNTER_MAP_OUTPUT_RECORDS = - "MAP_OUTPUT_RECORDS"; - - /** - * The name of the counter that tracks input records to the Map phase. - */ + org.apache.sqoop.config.ConfigurationConstants.COUNTER_MAP_OUTPUT_RECORDS; public static final String COUNTER_MAP_INPUT_RECORDS = - "MAP_INPUT_RECORDS"; + org.apache.sqoop.config.ConfigurationConstants.COUNTER_MAP_INPUT_RECORDS; private ConfigurationConstants() { - // Disable Explicit Object Creation } + } diff --git a/src/java/com/cloudera/sqoop/config/ConfigurationHelper.java b/src/java/com/cloudera/sqoop/config/ConfigurationHelper.java index 4fadfe97..08704977 100644 --- a/src/java/com/cloudera/sqoop/config/ConfigurationHelper.java +++ b/src/java/com/cloudera/sqoop/config/ConfigurationHelper.java @@ -1,6 +1,4 @@ /** - * Copyright 2011 The Apache Software Foundation - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.cloudera.sqoop.config; import java.io.IOException; @@ -25,149 +22,91 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.util.GenericOptionsParser; - -import com.cloudera.sqoop.mapreduce.db.DBConfiguration; /** - * This class provides static helper methods that allow access and manipulation - * of job configuration. It is convenient to keep such access in one place in - * order to allow easy modifications when some of these aspects change from - * version to version of Hadoop. + * @deprecated Moving to use org.apache.sqoop namespace. */ public final class ConfigurationHelper { - /** - * Set the (hinted) number of map tasks for a job. - */ public static void setJobNumMaps(Job job, int numMapTasks) { - job.getConfiguration().setInt( - ConfigurationConstants.PROP_MAPRED_MAP_TASKS, numMapTasks); + org.apache.sqoop.config.ConfigurationHelper.setJobNumMaps(job, numMapTasks); } - /** - * Get the (hinted) number of map tasks for a job. - */ public static int getJobNumMaps(JobContext job) { - return job.getConfiguration().getInt( - ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1); + return org.apache.sqoop.config.ConfigurationHelper.getJobNumMaps(job); } - /** - * @return the number of mapper output records from a job using its counters. - */ public static long getNumMapOutputRecords(Job job) throws IOException, InterruptedException { - return job.getCounters().findCounter( - ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS, - ConfigurationConstants.COUNTER_MAP_OUTPUT_RECORDS).getValue(); + return org.apache.sqoop.config. + ConfigurationHelper.getNumMapOutputRecords(job); } - /** - * @return the number of mapper input records from a job using its counters. - */ public static long getNumMapInputRecords(Job job) throws IOException, InterruptedException { - return job.getCounters().findCounter( - ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS, - ConfigurationConstants.COUNTER_MAP_INPUT_RECORDS).getValue(); + return org.apache.sqoop.config. + ConfigurationHelper.getNumMapInputRecords(job); } - /** - * Get the (hinted) number of map tasks for a job. - */ public static int getConfNumMaps(Configuration conf) { - return conf.getInt(ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1); + return org.apache.sqoop.config.ConfigurationHelper.getConfNumMaps(conf); } - /** - * Set the mapper speculative execution property for a job. - */ public static void setJobMapSpeculativeExecution(Job job, boolean isEnabled) { - job.getConfiguration().setBoolean( - ConfigurationConstants.PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC, - isEnabled); + org.apache.sqoop.config. + ConfigurationHelper.setJobMapSpeculativeExecution(job, isEnabled); } - /** - * Set the reducer speculative execution property for a job. - */ public static void setJobReduceSpeculativeExecution( Job job, boolean isEnabled) { - job.getConfiguration().setBoolean( - ConfigurationConstants.PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC, - isEnabled); + org.apache.sqoop.config. + ConfigurationHelper.setJobReduceSpeculativeExecution(job, isEnabled); } - /** - * Sets the Jobtracker address to use for a job. - */ public static void setJobtrackerAddr(Configuration conf, String addr) { - conf.set(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS, addr); + org.apache.sqoop.config. + ConfigurationHelper.setJobtrackerAddr(conf, addr); } - /** - * @return the Configuration property identifying a DBWritable to use. - */ public static String getDbInputClassProperty() { - return DBConfiguration.INPUT_CLASS_PROPERTY; + return org.apache.sqoop.config. + ConfigurationHelper.getDbInputClassProperty(); } - /** - * @return the Configuration property identifying the DB username. - */ public static String getDbUsernameProperty() { - return DBConfiguration.USERNAME_PROPERTY; + return org.apache.sqoop.config. + ConfigurationHelper.getDbUsernameProperty(); } - /** - * @return the Configuration property identifying the DB password. - */ public static String getDbPasswordProperty() { - return DBConfiguration.PASSWORD_PROPERTY; + return org.apache.sqoop.config. + ConfigurationHelper.getDbPasswordProperty(); } - /** - * @return the Configuration property identifying the DB connect string. - */ public static String getDbUrlProperty() { - return DBConfiguration.URL_PROPERTY; + return org.apache.sqoop.config. + ConfigurationHelper.getDbUrlProperty(); } - /** - * @return the Configuration property identifying the DB input table. - */ public static String getDbInputTableNameProperty() { - return DBConfiguration.INPUT_TABLE_NAME_PROPERTY; + return org.apache.sqoop.config. + ConfigurationHelper.getDbInputTableNameProperty(); } - /** - * @return the Configuration property specifying WHERE conditions for the - * db table. - */ + public static String getDbInputConditionsProperty() { - return DBConfiguration.INPUT_CONDITIONS_PROPERTY; + return org.apache.sqoop.config. + ConfigurationHelper.getDbInputConditionsProperty(); } - /** - * Parse arguments in 'args' via the GenericOptionsParser and - * embed the results in the supplied configuration. - * @param conf the configuration to populate with generic options. - * @param args the arguments to process. - * @return the unused args to be passed to the application itself. - */ public static String [] parseGenericOptions( Configuration conf, String [] args) throws IOException { - // This needs to be shimmed because in Apache Hadoop this can throw - // an IOException, but it does not do so in CDH. We just mandate in - // this method that an IOException is possible. - GenericOptionsParser genericParser = new GenericOptionsParser( - conf, args); - return genericParser.getRemainingArgs(); + return org.apache.sqoop.config. + ConfigurationHelper.parseGenericOptions(conf, args); } - private ConfigurationHelper() { // Disable explicit object creation } + } diff --git a/src/java/org/apache/sqoop/cli/RelatedOptions.java b/src/java/org/apache/sqoop/cli/RelatedOptions.java new file mode 100644 index 00000000..7e859c2a --- /dev/null +++ b/src/java/org/apache/sqoop/cli/RelatedOptions.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.cli; + +import org.apache.commons.cli.Options; + +/** + * Class that extends a set of options to provide a title for a + * related set of options. + */ +public class RelatedOptions extends Options { + + private String relatedTitle; + + public RelatedOptions() { + this(""); + } + + public RelatedOptions(final String title) { + super(); + this.relatedTitle = title; + } + + public String getTitle() { + return relatedTitle; + } + + @Override + public String toString() { + return relatedTitle + "\n" + super.toString(); + } +} + diff --git a/src/java/org/apache/sqoop/cli/SqoopParser.java b/src/java/org/apache/sqoop/cli/SqoopParser.java new file mode 100644 index 00000000..a70bf3c0 --- /dev/null +++ b/src/java/org/apache/sqoop/cli/SqoopParser.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.cli; + +import java.util.ListIterator; + +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.ParseException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.StringUtils; + +/** + * Options parser that follows Sqoop's specific options-parsing needs. + *
    + *
  • Option values may contain the '"' character as the first + * or last character in the value.
    • + *
  • The argument '--' must be preserved in the returned (unparsed) + * argument list.
    • + *
+ */ +public class SqoopParser extends GnuParser { + + public static final Log LOG = LogFactory.getLog(SqoopParser.class.getName()); + + // We need a handle to Option.addValueForProcessing(String). + // Since Hadoop will load this in a different classloader than + // this Sqoop class, we cannot see their package-specific methods. + // So we just call it by reflection. As long as we're at it, this + // allows us to also put SqoopParser in its own package. + private static java.lang.reflect.Method addValForProcessing; + + static { + try { + addValForProcessing = Option.class.getDeclaredMethod( + "addValueForProcessing", String.class); + addValForProcessing.setAccessible(true); + } catch (NoSuchMethodException nsme) { + LOG.error("Could not load required method of Parser: " + + StringUtils.stringifyException(nsme)); + addValForProcessing = null; + } + } + + @Override + /** + * Processes arguments to options but only strips matched quotes. + */ + public void processArgs(Option opt, ListIterator iter) + throws ParseException { + // Loop until an option is found. + while (iter.hasNext()) { + String str = (String) iter.next(); + + if (getOptions().hasOption(str) && str.startsWith("-")) { + // found an Option, not an argument. + iter.previous(); + break; + } + + // Otherwise, this is a value. + try { + // Note that we only strip matched quotes here. + addValForProcessing.invoke(opt, stripMatchedQuotes(str)); + } catch (IllegalAccessException iae) { + throw new RuntimeException(iae); + } catch (java.lang.reflect.InvocationTargetException ite) { + // Any runtime exception thrown within addValForProcessing() + // will be wrapped in an InvocationTargetException. + iter.previous(); + break; + } catch (RuntimeException re) { + iter.previous(); + break; + } + } + + if (opt.getValues() == null && !opt.hasOptionalArg()) { + throw new MissingArgumentException(opt); + } + } + + /** + * Util.stripLeadingAndTrailingQuotes() will strip a '"' + * character from either or both sides of a string. We only + * strip the matched pair. + */ + private String stripMatchedQuotes(String in) { + if (null == in || "\"".equals(in)) { + return in; // single quote stays as-is. + } else if(in.startsWith("\"") && in.endsWith("\"")) { + // Strip this pair of matched quotes. + return in.substring(1, in.length() - 1); + } else { + // return as-is. + return in; + } + } +} + diff --git a/src/java/org/apache/sqoop/cli/ToolOptions.java b/src/java/org/apache/sqoop/cli/ToolOptions.java new file mode 100644 index 00000000..f1488970 --- /dev/null +++ b/src/java/org/apache/sqoop/cli/ToolOptions.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.cli; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.cloudera.sqoop.cli.RelatedOptions; + +/** + * Class that holds several sets of related options, providing a container + * for all the options associated with a single tool. + * The order in which sets of related options are added to this tool is + * preserved in printing and iteration. + */ +public class ToolOptions implements Iterable { + + private List optGroups; + + public ToolOptions() { + this.optGroups = new ArrayList(); + } + + /** + * Add a block of related options to the options for this tool. + * @param opts the set of RelatedOptions to add. + */ + public void addOptions(RelatedOptions opts) { + optGroups.add(opts); + } + + /** + * Add a block of related options to the options for this tool, + * if a block has not already been added with the same title. + * @param opts the set of RelatedOptions to add. + */ + public void addUniqueOptions(RelatedOptions opts) { + if (!containsGroup(opts.getTitle())) { + optGroups.add(opts); + } + } + + /** + * Reports whether this collection of RelatedOptions contains + * a RelatedOptions with the specified title. + * @param title the group title to search for + * @return true if a RelatedOptions with this group title is + * in the collection. + */ + public boolean containsGroup(String title) { + for (RelatedOptions related : this) { + if (related.getTitle().equals(title)) { + return true; + } + } + + return false; + } + + /** + * Provide an iterator over all sets of RelatedOptions. + * @return an iterator returning each RelatedOptions element. + */ + public Iterator iterator() { + return optGroups.iterator(); + } + + + /** + * Flatten the different sets of related options into a single collection + * of options. + * @return all options in the ToolOptions as a single set + */ + public Options merge() { + Options mergedOpts = new Options(); + int totalOpts = 0; + for (RelatedOptions relatedOpts : this) { + for (Object optObj : relatedOpts.getOptions()) { + Option opt = (Option) optObj; + mergedOpts.addOption(opt); + totalOpts++; + } + } + + return mergedOpts; + } + + /** + * Print the help to the console using a default help formatter. + */ + public void printHelp() { + printHelp(new HelpFormatter()); + } + + /** + * Print the help to the console using the specified help formatter. + * @param formatter the HelpFormatter to use. + */ + public void printHelp(HelpFormatter formatter) { + printHelp(formatter, new PrintWriter(System.out, true)); + } + + /** + * Print the help to the specified PrintWriter, using the specified + * help formatter. + * @param formatter the HelpFormatter to use. + * @param pw the PrintWriter to emit to. + */ + public void printHelp(HelpFormatter formatter, PrintWriter pw) { + boolean first = true; + for (RelatedOptions optGroup : optGroups) { + if (!first) { + pw.println(""); + } + pw.println(optGroup.getTitle() + ":"); + formatter.printOptions(pw, formatter.getWidth(), optGroup, 0, 4); + first = false; + } + } + + @Override + public String toString() { + StringWriter sw = new StringWriter(); + printHelp(new HelpFormatter(), new PrintWriter(sw)); + sw.flush(); + return sw.getBuffer().toString(); + } + +} + diff --git a/src/java/org/apache/sqoop/config/ConfigurationConstants.java b/src/java/org/apache/sqoop/config/ConfigurationConstants.java new file mode 100644 index 00000000..53540635 --- /dev/null +++ b/src/java/org/apache/sqoop/config/ConfigurationConstants.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sqoop.config; + +/** + * Static constants that identify configuration keys, counter group names, and + * counter names. + */ +public final class ConfigurationConstants { + + /** + * The Configuration property identifying the current task id. + */ + public static final String PROP_MAPRED_TASK_ID = "mapred.task.id"; + + /** + * The Configuration property identifying the job's local directory. + */ + public static final String PROP_JOB_LOCAL_DIRECTORY = "job.local.dir"; + + /** + * The Configuration property identifying the number of map tasks to be used. + */ + public static final String PROP_MAPRED_MAP_TASKS = "mapred.map.tasks"; + + /** + * The Configuration property identifying the speculative execution flag for + * map tasks. + */ + public static final String PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC = + "mapred.map.tasks.speculative.execution"; + + /** + * The Configuration property identifying the speculative execution flag for + * reduce tasks. + */ + public static final String PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC = + "mapred.reduce.tasks.speculative.execution"; + + /** + * The Configuration property identifying the job tracker address. + */ + public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS = + "mapred.job.tracker"; + + /** + * The group name of task counters. + */ + public static final String COUNTER_GROUP_MAPRED_TASK_COUNTERS = + "org.apache.hadoop.mapred.Task$Counter"; + + /** + * The name of the counter that tracks output records from Map phase. + */ + public static final String COUNTER_MAP_OUTPUT_RECORDS = + "MAP_OUTPUT_RECORDS"; + + /** + * The name of the counter that tracks input records to the Map phase. + */ + public static final String COUNTER_MAP_INPUT_RECORDS = + "MAP_INPUT_RECORDS"; + + private ConfigurationConstants() { + // Disable Explicit Object Creation + } +} diff --git a/src/java/org/apache/sqoop/config/ConfigurationHelper.java b/src/java/org/apache/sqoop/config/ConfigurationHelper.java new file mode 100644 index 00000000..a4fb1edb --- /dev/null +++ b/src/java/org/apache/sqoop/config/ConfigurationHelper.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.config; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.util.GenericOptionsParser; + +import com.cloudera.sqoop.mapreduce.db.DBConfiguration; + +/** + * This class provides static helper methods that allow access and manipulation + * of job configuration. It is convenient to keep such access in one place in + * order to allow easy modifications when some of these aspects change from + * version to version of Hadoop. + */ +public final class ConfigurationHelper { + + /** + * Set the (hinted) number of map tasks for a job. + */ + public static void setJobNumMaps(Job job, int numMapTasks) { + job.getConfiguration().setInt( + ConfigurationConstants.PROP_MAPRED_MAP_TASKS, numMapTasks); + } + + /** + * Get the (hinted) number of map tasks for a job. + */ + public static int getJobNumMaps(JobContext job) { + return job.getConfiguration().getInt( + ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1); + } + + /** + * @return the number of mapper output records from a job using its counters. + */ + public static long getNumMapOutputRecords(Job job) + throws IOException, InterruptedException { + return job.getCounters().findCounter( + ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS, + ConfigurationConstants.COUNTER_MAP_OUTPUT_RECORDS).getValue(); + } + + /** + * @return the number of mapper input records from a job using its counters. + */ + public static long getNumMapInputRecords(Job job) + throws IOException, InterruptedException { + return job.getCounters().findCounter( + ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS, + ConfigurationConstants.COUNTER_MAP_INPUT_RECORDS).getValue(); + } + + /** + * Get the (hinted) number of map tasks for a job. + */ + public static int getConfNumMaps(Configuration conf) { + return conf.getInt(ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1); + } + + /** + * Set the mapper speculative execution property for a job. + */ + public static void setJobMapSpeculativeExecution(Job job, boolean isEnabled) { + job.getConfiguration().setBoolean( + ConfigurationConstants.PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC, + isEnabled); + } + + /** + * Set the reducer speculative execution property for a job. + */ + public static void setJobReduceSpeculativeExecution( + Job job, boolean isEnabled) { + job.getConfiguration().setBoolean( + ConfigurationConstants.PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC, + isEnabled); + } + + /** + * Sets the Jobtracker address to use for a job. + */ + public static void setJobtrackerAddr(Configuration conf, String addr) { + conf.set(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS, addr); + } + + /** + * @return the Configuration property identifying a DBWritable to use. + */ + public static String getDbInputClassProperty() { + return DBConfiguration.INPUT_CLASS_PROPERTY; + } + + /** + * @return the Configuration property identifying the DB username. + */ + public static String getDbUsernameProperty() { + return DBConfiguration.USERNAME_PROPERTY; + } + + /** + * @return the Configuration property identifying the DB password. + */ + public static String getDbPasswordProperty() { + return DBConfiguration.PASSWORD_PROPERTY; + } + + /** + * @return the Configuration property identifying the DB connect string. + */ + public static String getDbUrlProperty() { + return DBConfiguration.URL_PROPERTY; + } + + /** + * @return the Configuration property identifying the DB input table. + */ + public static String getDbInputTableNameProperty() { + return DBConfiguration.INPUT_TABLE_NAME_PROPERTY; + } + + /** + * @return the Configuration property specifying WHERE conditions for the + * db table. + */ + public static String getDbInputConditionsProperty() { + return DBConfiguration.INPUT_CONDITIONS_PROPERTY; + } + + /** + * Parse arguments in 'args' via the GenericOptionsParser and + * embed the results in the supplied configuration. + * @param conf the configuration to populate with generic options. + * @param args the arguments to process. + * @return the unused args to be passed to the application itself. + */ + public static String [] parseGenericOptions( + Configuration conf, String [] args) throws IOException { + // This needs to be shimmed because in Apache Hadoop this can throw + // an IOException, but it does not do so in CDH. We just mandate in + // this method that an IOException is possible. + GenericOptionsParser genericParser = new GenericOptionsParser( + conf, args); + return genericParser.getRemainingArgs(); + } + + + private ConfigurationHelper() { + // Disable explicit object creation + } +} diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlTest.java index 574003d0..ecefee8e 100644 --- a/src/test/com/cloudera/sqoop/manager/PostgresqlTest.java +++ b/src/test/com/cloudera/sqoop/manager/PostgresqlTest.java @@ -178,7 +178,8 @@ public void setUpData(String tableName) { } - private String [] getArgv(boolean isDirect, String tableName) { + private String [] getArgv(boolean isDirect, String tableName, + String... extraArgs) { ArrayList args = new ArrayList(); CommonArgs.addHadoopFlags(args); @@ -198,11 +199,15 @@ public void setUpData(String tableName) { args.add("--direct"); } + for (String arg : extraArgs) { + args.add(arg); + } + return args.toArray(new String[0]); } private void doImportAndVerify(boolean isDirect, String [] expectedResults, - String tableName) throws IOException { + String tableName, String... extraArgs) throws IOException { Path warehousePath = new Path(this.getWarehouseDir()); Path tablePath = new Path(warehousePath, tableName); @@ -220,7 +225,7 @@ private void doImportAndVerify(boolean isDirect, String [] expectedResults, FileListing.recursiveDeleteDir(tableFile); } - String [] argv = getArgv(isDirect, tableName); + String [] argv = getArgv(isDirect, tableName, extraArgs); try { runImport(argv); } catch (IOException ioe) { @@ -289,4 +294,15 @@ public void testTableNameWithSpecialCharacter() throws IOException { doImportAndVerify(false, expectedResults, SPECIAL_TABLE_NAME); } + + @Test + public void testIncrementalImport() throws IOException { + String [] expectedResults = { }; + + String [] extraArgs = { "--incremental", "lastmodified", + "--check-column", "start_date", + }; + + doImportAndVerify(false, expectedResults, TABLE_NAME, extraArgs); + } }