github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 19:21:37 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-381: Migrate cli and config packages to new name space

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1195865 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Bilung Lee 2011-11-01 08:20:12 +00:00
parent 916983e137
commit fb1c2c94c4
11 changed files with 653 additions and 395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/java/com/cloudera/sqoop/cli/RelatedOptions.java
View File

@ -1,6 +1,4 @@
/** /**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
@ -17,35 +15,20 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package com.cloudera.sqoop.cli; package com.cloudera.sqoop.cli;
import org.apache.commons.cli.Options;
/** /**
* Class that extends a set of options to provide a title for a * @deprecated Moving to use org.apache.sqoop namespace.
* related set of options.
*/ */
public class RelatedOptions extends Options { public class RelatedOptions
extends org.apache.sqoop.cli.RelatedOptions {
private String relatedTitle;
public RelatedOptions() { public RelatedOptions() {
this(""); super();
} }
public RelatedOptions(final String title) { public RelatedOptions(final String title) {
super(); super(title);
this.relatedTitle = title;
}
public String getTitle() {
return relatedTitle;
}
@Override
public String toString() {
return relatedTitle + "\n" + super.toString();
} }
} }

100
src/java/com/cloudera/sqoop/cli/SqoopParser.java
View File

@ -1,6 +1,4 @@
/** /**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
@ -17,104 +15,12 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package com.cloudera.sqoop.cli; package com.cloudera.sqoop.cli;
import java.util.ListIterator;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
/** /**
* Options parser that follows Sqoop's specific options-parsing needs. * @deprecated Moving to use org.apache.sqoop namespace.
* <ul>
* <li>Option values may contain the '&quot;' character as the first
* or last character in the value.</li>
* <li>The argument '--' must be preserved in the returned (unparsed)
* argument list.</li>
* </ul>
*/ */
public class SqoopParser extends GnuParser { public class SqoopParser
extends org.apache.sqoop.cli.SqoopParser {
public static final Log LOG = LogFactory.getLog(SqoopParser.class.getName());
// We need a handle to Option.addValueForProcessing(String).
// Since Hadoop will load this in a different classloader than
// this Sqoop class, we cannot see their package-specific methods.
// So we just call it by reflection. As long as we're at it, this
// allows us to also put SqoopParser in its own package.
private static java.lang.reflect.Method addValForProcessing;
static {
try {
addValForProcessing = Option.class.getDeclaredMethod(
"addValueForProcessing", String.class);
addValForProcessing.setAccessible(true);
} catch (NoSuchMethodException nsme) {
LOG.error("Could not load required method of Parser: "
+ StringUtils.stringifyException(nsme));
addValForProcessing = null;
}
}
@Override
/**
* Processes arguments to options but only strips matched quotes.
*/
public void processArgs(Option opt, ListIterator iter)
throws ParseException {
// Loop until an option is found.
while (iter.hasNext()) {
String str = (String) iter.next();
if (getOptions().hasOption(str) && str.startsWith("-")) {
// found an Option, not an argument.
iter.previous();
break;
}
// Otherwise, this is a value.
try {
// Note that we only strip matched quotes here.
addValForProcessing.invoke(opt, stripMatchedQuotes(str));
} catch (IllegalAccessException iae) {
throw new RuntimeException(iae);
} catch (java.lang.reflect.InvocationTargetException ite) {
// Any runtime exception thrown within addValForProcessing()
// will be wrapped in an InvocationTargetException.
iter.previous();
break;
} catch (RuntimeException re) {
iter.previous();
break;
}
}
if (opt.getValues() == null && !opt.hasOptionalArg()) {
throw new MissingArgumentException(opt);
}
}
/**
* Util.stripLeadingAndTrailingQuotes() will strip a '&quot;'
* character from either or both sides of a string. We only
* strip the matched pair.
*/
private String stripMatchedQuotes(String in) {
if (null == in || "\"".equals(in)) {
return in; // single quote stays as-is.
} else if(in.startsWith("\"") && in.endsWith("\"")) {
// Strip this pair of matched quotes.
return in.substring(1, in.length() - 1);
} else {
// return as-is.
return in;
}
}
} }

132
src/java/com/cloudera/sqoop/cli/ToolOptions.java
View File

@ -1,6 +1,4 @@
/** /**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
@ -20,134 +18,10 @@
package com.cloudera.sqoop.cli; package com.cloudera.sqoop.cli;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
/** /**
* Class that holds several sets of related options, providing a container * @deprecated Moving to use org.apache.sqoop namespace.
* for all the options associated with a single tool.
* The order in which sets of related options are added to this tool is
* preserved in printing and iteration.
*/ */
public class ToolOptions implements Iterable<RelatedOptions> { public class ToolOptions
extends org.apache.sqoop.cli.ToolOptions {
private List<RelatedOptions> optGroups;
public ToolOptions() {
this.optGroups = new ArrayList<RelatedOptions>();
}
/**
* Add a block of related options to the options for this tool.
* @param opts the set of RelatedOptions to add.
*/
public void addOptions(RelatedOptions opts) {
optGroups.add(opts);
}
/**
* Add a block of related options to the options for this tool,
* if a block has not already been added with the same title.
* @param opts the set of RelatedOptions to add.
*/
public void addUniqueOptions(RelatedOptions opts) {
if (!containsGroup(opts.getTitle())) {
optGroups.add(opts);
}
}
/**
* Reports whether this collection of RelatedOptions contains
* a RelatedOptions with the specified title.
* @param title the group title to search for
* @return true if a RelatedOptions with this group title is
* in the collection.
*/
public boolean containsGroup(String title) {
for (RelatedOptions related : this) {
if (related.getTitle().equals(title)) {
return true;
}
}
return false;
}
/**
* Provide an iterator over all sets of RelatedOptions.
* @return an iterator returning each RelatedOptions element.
*/
public Iterator<RelatedOptions> iterator() {
return optGroups.iterator();
}
/**
* Flatten the different sets of related options into a single collection
* of options.
* @return all options in the ToolOptions as a single set
*/
public Options merge() {
Options mergedOpts = new Options();
int totalOpts = 0;
for (RelatedOptions relatedOpts : this) {
for (Object optObj : relatedOpts.getOptions()) {
Option opt = (Option) optObj;
mergedOpts.addOption(opt);
totalOpts++;
}
}
return mergedOpts;
}
/**
* Print the help to the console using a default help formatter.
*/
public void printHelp() {
printHelp(new HelpFormatter());
}
/**
* Print the help to the console using the specified help formatter.
* @param formatter the HelpFormatter to use.
*/
public void printHelp(HelpFormatter formatter) {
printHelp(formatter, new PrintWriter(System.out, true));
}
/**
* Print the help to the specified PrintWriter, using the specified
* help formatter.
* @param formatter the HelpFormatter to use.
* @param pw the PrintWriter to emit to.
*/
public void printHelp(HelpFormatter formatter, PrintWriter pw) {
boolean first = true;
for (RelatedOptions optGroup : optGroups) {
if (!first) {
pw.println("");
}
pw.println(optGroup.getTitle() + ":");
formatter.printOptions(pw, formatter.getWidth(), optGroup, 0, 4);
first = false;
}
}
@Override
public String toString() {
StringWriter sw = new StringWriter();
printHelp(new HelpFormatter(), new PrintWriter(sw));
sw.flush();
return sw.getBuffer().toString();
}
} }

71
src/java/com/cloudera/sqoop/config/ConfigurationConstants.java
View File

@ -1,6 +1,4 @@
/** /**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
@ -17,70 +15,37 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package com.cloudera.sqoop.config; package com.cloudera.sqoop.config;
/** /**
* Static constants that identify configuration keys, counter group names, and * @deprecated Moving to use org.apache.sqoop namespace.
* counter names.
*/ */
public final class ConfigurationConstants { public final class ConfigurationConstants {
/** public static final String PROP_MAPRED_TASK_ID =
* The Configuration property identifying the current task id. org.apache.sqoop.config.ConfigurationConstants.PROP_MAPRED_TASK_ID;
*/ public static final String PROP_JOB_LOCAL_DIRECTORY =
public static final String PROP_MAPRED_TASK_ID = "mapred.task.id"; org.apache.sqoop.config.ConfigurationConstants.PROP_JOB_LOCAL_DIRECTORY;
public static final String PROP_MAPRED_MAP_TASKS =
/** org.apache.sqoop.config.ConfigurationConstants.PROP_MAPRED_MAP_TASKS;
* The Configuration property identifying the job's local directory.
*/
public static final String PROP_JOB_LOCAL_DIRECTORY = "job.local.dir";
/**
* The Configuration property identifying the number of map tasks to be used.
*/
public static final String PROP_MAPRED_MAP_TASKS = "mapred.map.tasks";
/**
* The Configuration property identifying the speculative execution flag for
* map tasks.
*/
public static final String PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC = public static final String PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC =
"mapred.map.tasks.speculative.execution"; org.apache.sqoop.config.
ConfigurationConstants.PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC;
/**
* The Configuration property identifying the speculative execution flag for
* reduce tasks.
*/
public static final String PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC = public static final String PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC =
"mapred.reduce.tasks.speculative.execution"; org.apache.sqoop.config.
ConfigurationConstants.PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC;
/**
* The Configuration property identifying the job tracker address.
*/
public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS = public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS =
"mapred.job.tracker"; org.apache.sqoop.config.
ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS;
/**
* The group name of task counters.
*/
public static final String COUNTER_GROUP_MAPRED_TASK_COUNTERS = public static final String COUNTER_GROUP_MAPRED_TASK_COUNTERS =
"org.apache.hadoop.mapred.Task$Counter"; org.apache.sqoop.config.
ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS;
/**
* The name of the counter that tracks output records from Map phase.
*/
public static final String COUNTER_MAP_OUTPUT_RECORDS = public static final String COUNTER_MAP_OUTPUT_RECORDS =
"MAP_OUTPUT_RECORDS"; org.apache.sqoop.config.ConfigurationConstants.COUNTER_MAP_OUTPUT_RECORDS;
/**
* The name of the counter that tracks input records to the Map phase.
*/
public static final String COUNTER_MAP_INPUT_RECORDS = public static final String COUNTER_MAP_INPUT_RECORDS =
"MAP_INPUT_RECORDS"; org.apache.sqoop.config.ConfigurationConstants.COUNTER_MAP_INPUT_RECORDS;
private ConfigurationConstants() { private ConfigurationConstants() {
// Disable Explicit Object Creation
} }
} }

121
src/java/com/cloudera/sqoop/config/ConfigurationHelper.java
View File

@ -1,6 +1,4 @@
/** /**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
@ -17,7 +15,6 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package com.cloudera.sqoop.config; package com.cloudera.sqoop.config;
import java.io.IOException; import java.io.IOException;
@ -25,149 +22,91 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.util.GenericOptionsParser;
import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
/** /**
* This class provides static helper methods that allow access and manipulation * @deprecated Moving to use org.apache.sqoop namespace.
* of job configuration. It is convenient to keep such access in one place in
* order to allow easy modifications when some of these aspects change from
* version to version of Hadoop.
*/ */
public final class ConfigurationHelper { public final class ConfigurationHelper {
/**
* Set the (hinted) number of map tasks for a job.
*/
public static void setJobNumMaps(Job job, int numMapTasks) { public static void setJobNumMaps(Job job, int numMapTasks) {
job.getConfiguration().setInt( org.apache.sqoop.config.ConfigurationHelper.setJobNumMaps(job, numMapTasks);
ConfigurationConstants.PROP_MAPRED_MAP_TASKS, numMapTasks);
} }
/**
* Get the (hinted) number of map tasks for a job.
*/
public static int getJobNumMaps(JobContext job) { public static int getJobNumMaps(JobContext job) {
return job.getConfiguration().getInt( return org.apache.sqoop.config.ConfigurationHelper.getJobNumMaps(job);
ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1);
} }
/**
* @return the number of mapper output records from a job using its counters.
*/
public static long getNumMapOutputRecords(Job job) public static long getNumMapOutputRecords(Job job)
throws IOException, InterruptedException { throws IOException, InterruptedException {
return job.getCounters().findCounter( return org.apache.sqoop.config.
ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS, ConfigurationHelper.getNumMapOutputRecords(job);
ConfigurationConstants.COUNTER_MAP_OUTPUT_RECORDS).getValue();
} }
/**
* @return the number of mapper input records from a job using its counters.
*/
public static long getNumMapInputRecords(Job job) public static long getNumMapInputRecords(Job job)
throws IOException, InterruptedException { throws IOException, InterruptedException {
return job.getCounters().findCounter( return org.apache.sqoop.config.
ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS, ConfigurationHelper.getNumMapInputRecords(job);
ConfigurationConstants.COUNTER_MAP_INPUT_RECORDS).getValue();
} }
/**
* Get the (hinted) number of map tasks for a job.
*/
public static int getConfNumMaps(Configuration conf) { public static int getConfNumMaps(Configuration conf) {
return conf.getInt(ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1); return org.apache.sqoop.config.ConfigurationHelper.getConfNumMaps(conf);
} }
/**
* Set the mapper speculative execution property for a job.
*/
public static void setJobMapSpeculativeExecution(Job job, boolean isEnabled) { public static void setJobMapSpeculativeExecution(Job job, boolean isEnabled) {
job.getConfiguration().setBoolean( org.apache.sqoop.config.
ConfigurationConstants.PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC, ConfigurationHelper.setJobMapSpeculativeExecution(job, isEnabled);
isEnabled);
} }
/**
* Set the reducer speculative execution property for a job.
*/
public static void setJobReduceSpeculativeExecution( public static void setJobReduceSpeculativeExecution(
Job job, boolean isEnabled) { Job job, boolean isEnabled) {
job.getConfiguration().setBoolean( org.apache.sqoop.config.
ConfigurationConstants.PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC, ConfigurationHelper.setJobReduceSpeculativeExecution(job, isEnabled);
isEnabled);
} }
/**
* Sets the Jobtracker address to use for a job.
*/
public static void setJobtrackerAddr(Configuration conf, String addr) { public static void setJobtrackerAddr(Configuration conf, String addr) {
conf.set(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS, addr); org.apache.sqoop.config.
ConfigurationHelper.setJobtrackerAddr(conf, addr);
} }
/**
* @return the Configuration property identifying a DBWritable to use.
*/
public static String getDbInputClassProperty() { public static String getDbInputClassProperty() {
return DBConfiguration.INPUT_CLASS_PROPERTY; return org.apache.sqoop.config.
ConfigurationHelper.getDbInputClassProperty();
} }
/**
* @return the Configuration property identifying the DB username.
*/
public static String getDbUsernameProperty() { public static String getDbUsernameProperty() {
return DBConfiguration.USERNAME_PROPERTY; return org.apache.sqoop.config.
ConfigurationHelper.getDbUsernameProperty();
} }
/**
* @return the Configuration property identifying the DB password.
*/
public static String getDbPasswordProperty() { public static String getDbPasswordProperty() {
return DBConfiguration.PASSWORD_PROPERTY; return org.apache.sqoop.config.
ConfigurationHelper.getDbPasswordProperty();
} }
/**
* @return the Configuration property identifying the DB connect string.
*/
public static String getDbUrlProperty() { public static String getDbUrlProperty() {
return DBConfiguration.URL_PROPERTY; return org.apache.sqoop.config.
ConfigurationHelper.getDbUrlProperty();
} }
/**
* @return the Configuration property identifying the DB input table.
*/
public static String getDbInputTableNameProperty() { public static String getDbInputTableNameProperty() {
return DBConfiguration.INPUT_TABLE_NAME_PROPERTY; return org.apache.sqoop.config.
ConfigurationHelper.getDbInputTableNameProperty();
} }
/**
* @return the Configuration property specifying WHERE conditions for the
* db table.
*/
public static String getDbInputConditionsProperty() { public static String getDbInputConditionsProperty() {
return DBConfiguration.INPUT_CONDITIONS_PROPERTY; return org.apache.sqoop.config.
ConfigurationHelper.getDbInputConditionsProperty();
} }
/**
* Parse arguments in 'args' via the GenericOptionsParser and
* embed the results in the supplied configuration.
* @param conf the configuration to populate with generic options.
* @param args the arguments to process.
* @return the unused args to be passed to the application itself.
*/
public static String [] parseGenericOptions( public static String [] parseGenericOptions(
Configuration conf, String [] args) throws IOException { Configuration conf, String [] args) throws IOException {
// This needs to be shimmed because in Apache Hadoop this can throw return org.apache.sqoop.config.
// an IOException, but it does not do so in CDH. We just mandate in ConfigurationHelper.parseGenericOptions(conf, args);
// this method that an IOException is possible.
GenericOptionsParser genericParser = new GenericOptionsParser(
conf, args);
return genericParser.getRemainingArgs();
} }
private ConfigurationHelper() { private ConfigurationHelper() {
// Disable explicit object creation // Disable explicit object creation
} }
} }

49
src/java/org/apache/sqoop/cli/RelatedOptions.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.cli;
import org.apache.commons.cli.Options;
/**
* Class that extends a set of options to provide a title for a
* related set of options.
*/
public class RelatedOptions extends Options {
private String relatedTitle;
public RelatedOptions() {
this("");
}
public RelatedOptions(final String title) {
super();
this.relatedTitle = title;
}
public String getTitle() {
return relatedTitle;
}
@Override
public String toString() {
return relatedTitle + "\n" + super.toString();
}
}

118
src/java/org/apache/sqoop/cli/SqoopParser.java Normal file
View File

@ -0,0 +1,118 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.cli;
import java.util.ListIterator;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
/**
* Options parser that follows Sqoop's specific options-parsing needs.
* <ul>
* <li>Option values may contain the '&quot;' character as the first
* or last character in the value.</li>
* <li>The argument '--' must be preserved in the returned (unparsed)
* argument list.</li>
* </ul>
*/
public class SqoopParser extends GnuParser {
public static final Log LOG = LogFactory.getLog(SqoopParser.class.getName());
// We need a handle to Option.addValueForProcessing(String).
// Since Hadoop will load this in a different classloader than
// this Sqoop class, we cannot see their package-specific methods.
// So we just call it by reflection. As long as we're at it, this
// allows us to also put SqoopParser in its own package.
private static java.lang.reflect.Method addValForProcessing;
static {
try {
addValForProcessing = Option.class.getDeclaredMethod(
"addValueForProcessing", String.class);
addValForProcessing.setAccessible(true);
} catch (NoSuchMethodException nsme) {
LOG.error("Could not load required method of Parser: "
+ StringUtils.stringifyException(nsme));
addValForProcessing = null;
}
}
@Override
/**
* Processes arguments to options but only strips matched quotes.
*/
public void processArgs(Option opt, ListIterator iter)
throws ParseException {
// Loop until an option is found.
while (iter.hasNext()) {
String str = (String) iter.next();
if (getOptions().hasOption(str) && str.startsWith("-")) {
// found an Option, not an argument.
iter.previous();
break;
}
// Otherwise, this is a value.
try {
// Note that we only strip matched quotes here.
addValForProcessing.invoke(opt, stripMatchedQuotes(str));
} catch (IllegalAccessException iae) {
throw new RuntimeException(iae);
} catch (java.lang.reflect.InvocationTargetException ite) {
// Any runtime exception thrown within addValForProcessing()
// will be wrapped in an InvocationTargetException.
iter.previous();
break;
} catch (RuntimeException re) {
iter.previous();
break;
}
}
if (opt.getValues() == null && !opt.hasOptionalArg()) {
throw new MissingArgumentException(opt);
}
}
/**
* Util.stripLeadingAndTrailingQuotes() will strip a '&quot;'
* character from either or both sides of a string. We only
* strip the matched pair.
*/
private String stripMatchedQuotes(String in) {
if (null == in || "\"".equals(in)) {
return in; // single quote stays as-is.
} else if(in.startsWith("\"") && in.endsWith("\"")) {
// Strip this pair of matched quotes.
return in.substring(1, in.length() - 1);
} else {
// return as-is.
return in;
}
}
}

153
src/java/org/apache/sqoop/cli/ToolOptions.java Normal file
View File

@ -0,0 +1,153 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.cli;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import com.cloudera.sqoop.cli.RelatedOptions;
/**
* Class that holds several sets of related options, providing a container
* for all the options associated with a single tool.
* The order in which sets of related options are added to this tool is
* preserved in printing and iteration.
*/
public class ToolOptions implements Iterable<RelatedOptions> {
private List<RelatedOptions> optGroups;
public ToolOptions() {
this.optGroups = new ArrayList<RelatedOptions>();
}
/**
* Add a block of related options to the options for this tool.
* @param opts the set of RelatedOptions to add.
*/
public void addOptions(RelatedOptions opts) {
optGroups.add(opts);
}
/**
* Add a block of related options to the options for this tool,
* if a block has not already been added with the same title.
* @param opts the set of RelatedOptions to add.
*/
public void addUniqueOptions(RelatedOptions opts) {
if (!containsGroup(opts.getTitle())) {
optGroups.add(opts);
}
}
/**
* Reports whether this collection of RelatedOptions contains
* a RelatedOptions with the specified title.
* @param title the group title to search for
* @return true if a RelatedOptions with this group title is
* in the collection.
*/
public boolean containsGroup(String title) {
for (RelatedOptions related : this) {
if (related.getTitle().equals(title)) {
return true;
}
}
return false;
}
/**
* Provide an iterator over all sets of RelatedOptions.
* @return an iterator returning each RelatedOptions element.
*/
public Iterator<RelatedOptions> iterator() {
return optGroups.iterator();
}
/**
* Flatten the different sets of related options into a single collection
* of options.
* @return all options in the ToolOptions as a single set
*/
public Options merge() {
Options mergedOpts = new Options();
int totalOpts = 0;
for (RelatedOptions relatedOpts : this) {
for (Object optObj : relatedOpts.getOptions()) {
Option opt = (Option) optObj;
mergedOpts.addOption(opt);
totalOpts++;
}
}
return mergedOpts;
}
/**
* Print the help to the console using a default help formatter.
*/
public void printHelp() {
printHelp(new HelpFormatter());
}
/**
* Print the help to the console using the specified help formatter.
* @param formatter the HelpFormatter to use.
*/
public void printHelp(HelpFormatter formatter) {
printHelp(formatter, new PrintWriter(System.out, true));
}
/**
* Print the help to the specified PrintWriter, using the specified
* help formatter.
* @param formatter the HelpFormatter to use.
* @param pw the PrintWriter to emit to.
*/
public void printHelp(HelpFormatter formatter, PrintWriter pw) {
boolean first = true;
for (RelatedOptions optGroup : optGroups) {
if (!first) {
pw.println("");
}
pw.println(optGroup.getTitle() + ":");
formatter.printOptions(pw, formatter.getWidth(), optGroup, 0, 4);
first = false;
}
}
@Override
public String toString() {
StringWriter sw = new StringWriter();
printHelp(new HelpFormatter(), new PrintWriter(sw));
sw.flush();
return sw.getBuffer().toString();
}
}

84
src/java/org/apache/sqoop/config/ConfigurationConstants.java Normal file
View File

@ -0,0 +1,84 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.config;
/**
* Static constants that identify configuration keys, counter group names, and
* counter names.
*/
public final class ConfigurationConstants {
/**
* The Configuration property identifying the current task id.
*/
public static final String PROP_MAPRED_TASK_ID = "mapred.task.id";
/**
* The Configuration property identifying the job's local directory.
*/
public static final String PROP_JOB_LOCAL_DIRECTORY = "job.local.dir";
/**
* The Configuration property identifying the number of map tasks to be used.
*/
public static final String PROP_MAPRED_MAP_TASKS = "mapred.map.tasks";
/**
* The Configuration property identifying the speculative execution flag for
* map tasks.
*/
public static final String PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC =
"mapred.map.tasks.speculative.execution";
/**
* The Configuration property identifying the speculative execution flag for
* reduce tasks.
*/
public static final String PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC =
"mapred.reduce.tasks.speculative.execution";
/**
* The Configuration property identifying the job tracker address.
*/
public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS =
"mapred.job.tracker";
/**
* The group name of task counters.
*/
public static final String COUNTER_GROUP_MAPRED_TASK_COUNTERS =
"org.apache.hadoop.mapred.Task$Counter";
/**
* The name of the counter that tracks output records from Map phase.
*/
public static final String COUNTER_MAP_OUTPUT_RECORDS =
"MAP_OUTPUT_RECORDS";
/**
* The name of the counter that tracks input records to the Map phase.
*/
public static final String COUNTER_MAP_INPUT_RECORDS =
"MAP_INPUT_RECORDS";
private ConfigurationConstants() {
// Disable Explicit Object Creation
}
}

171
src/java/org/apache/sqoop/config/ConfigurationHelper.java Normal file
View File

@ -0,0 +1,171 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.config;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.util.GenericOptionsParser;
import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
/**
* This class provides static helper methods that allow access and manipulation
* of job configuration. It is convenient to keep such access in one place in
* order to allow easy modifications when some of these aspects change from
* version to version of Hadoop.
*/
public final class ConfigurationHelper {
/**
* Set the (hinted) number of map tasks for a job.
*/
public static void setJobNumMaps(Job job, int numMapTasks) {
job.getConfiguration().setInt(
ConfigurationConstants.PROP_MAPRED_MAP_TASKS, numMapTasks);
}
/**
* Get the (hinted) number of map tasks for a job.
*/
public static int getJobNumMaps(JobContext job) {
return job.getConfiguration().getInt(
ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1);
}
/**
* @return the number of mapper output records from a job using its counters.
*/
public static long getNumMapOutputRecords(Job job)
throws IOException, InterruptedException {
return job.getCounters().findCounter(
ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS,
ConfigurationConstants.COUNTER_MAP_OUTPUT_RECORDS).getValue();
}
/**
* @return the number of mapper input records from a job using its counters.
*/
public static long getNumMapInputRecords(Job job)
throws IOException, InterruptedException {
return job.getCounters().findCounter(
ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS,
ConfigurationConstants.COUNTER_MAP_INPUT_RECORDS).getValue();
}
/**
* Get the (hinted) number of map tasks for a job.
*/
public static int getConfNumMaps(Configuration conf) {
return conf.getInt(ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1);
}
/**
* Set the mapper speculative execution property for a job.
*/
public static void setJobMapSpeculativeExecution(Job job, boolean isEnabled) {
job.getConfiguration().setBoolean(
ConfigurationConstants.PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC,
isEnabled);
}
/**
* Set the reducer speculative execution property for a job.
*/
public static void setJobReduceSpeculativeExecution(
Job job, boolean isEnabled) {
job.getConfiguration().setBoolean(
ConfigurationConstants.PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC,
isEnabled);
}
/**
* Sets the Jobtracker address to use for a job.
*/
public static void setJobtrackerAddr(Configuration conf, String addr) {
conf.set(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS, addr);
}
/**
* @return the Configuration property identifying a DBWritable to use.
*/
public static String getDbInputClassProperty() {
return DBConfiguration.INPUT_CLASS_PROPERTY;
}
/**
* @return the Configuration property identifying the DB username.
*/
public static String getDbUsernameProperty() {
return DBConfiguration.USERNAME_PROPERTY;
}
/**
* @return the Configuration property identifying the DB password.
*/
public static String getDbPasswordProperty() {
return DBConfiguration.PASSWORD_PROPERTY;
}
/**
* @return the Configuration property identifying the DB connect string.
*/
public static String getDbUrlProperty() {
return DBConfiguration.URL_PROPERTY;
}
/**
* @return the Configuration property identifying the DB input table.
*/
public static String getDbInputTableNameProperty() {
return DBConfiguration.INPUT_TABLE_NAME_PROPERTY;
}
/**
* @return the Configuration property specifying WHERE conditions for the
* db table.
*/
public static String getDbInputConditionsProperty() {
return DBConfiguration.INPUT_CONDITIONS_PROPERTY;
}
/**
* Parse arguments in 'args' via the GenericOptionsParser and
* embed the results in the supplied configuration.
* @param conf the configuration to populate with generic options.
* @param args the arguments to process.
* @return the unused args to be passed to the application itself.
*/
public static String [] parseGenericOptions(
Configuration conf, String [] args) throws IOException {
// This needs to be shimmed because in Apache Hadoop this can throw
// an IOException, but it does not do so in CDH. We just mandate in
// this method that an IOException is possible.
GenericOptionsParser genericParser = new GenericOptionsParser(
conf, args);
return genericParser.getRemainingArgs();
}
private ConfigurationHelper() {
// Disable explicit object creation
}
}

22
src/test/com/cloudera/sqoop/manager/PostgresqlTest.java
View File

@ -178,7 +178,8 @@ public void setUpData(String tableName) {
} }
private String [] getArgv(boolean isDirect, String tableName) { private String [] getArgv(boolean isDirect, String tableName,
String... extraArgs) {
ArrayList<String> args = new ArrayList<String>(); ArrayList<String> args = new ArrayList<String>();
CommonArgs.addHadoopFlags(args); CommonArgs.addHadoopFlags(args);
@ -198,11 +199,15 @@ public void setUpData(String tableName) {
args.add("--direct"); args.add("--direct");
} }
for (String arg : extraArgs) {
args.add(arg);
}
return args.toArray(new String[0]); return args.toArray(new String[0]);
} }
private void doImportAndVerify(boolean isDirect, String [] expectedResults, private void doImportAndVerify(boolean isDirect, String [] expectedResults,
String tableName) throws IOException { String tableName, String... extraArgs) throws IOException {
Path warehousePath = new Path(this.getWarehouseDir()); Path warehousePath = new Path(this.getWarehouseDir());
Path tablePath = new Path(warehousePath, tableName); Path tablePath = new Path(warehousePath, tableName);
@ -220,7 +225,7 @@ private void doImportAndVerify(boolean isDirect, String [] expectedResults,
FileListing.recursiveDeleteDir(tableFile); FileListing.recursiveDeleteDir(tableFile);
} }
String [] argv = getArgv(isDirect, tableName); String [] argv = getArgv(isDirect, tableName, extraArgs);
try { try {
runImport(argv); runImport(argv);
} catch (IOException ioe) { } catch (IOException ioe) {
@ -289,4 +294,15 @@ public void testTableNameWithSpecialCharacter() throws IOException {
doImportAndVerify(false, expectedResults, SPECIAL_TABLE_NAME); doImportAndVerify(false, expectedResults, SPECIAL_TABLE_NAME);
} }
@Test
public void testIncrementalImport() throws IOException {
String [] expectedResults = { };
String [] extraArgs = { "--incremental", "lastmodified",
"--check-column", "start_date",
};
doImportAndVerify(false, expectedResults, TABLE_NAME, extraArgs);
}
} }