github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 05:31:14 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-381: Migrate cli and config packages to new name space

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1195865 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Bilung Lee 2011-11-01 08:20:12 +00:00
parent 916983e137
commit fb1c2c94c4
11 changed files with 653 additions and 395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/java/com/cloudera/sqoop/cli/RelatedOptions.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -17,35 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloudera.sqoop.cli;
import org.apache.commons.cli.Options;
/**
* Class that extends a set of options to provide a title for a
* related set of options.
* @deprecated Moving to use org.apache.sqoop namespace.
*/
public class RelatedOptions extends Options {
private String relatedTitle;
public class RelatedOptions
extends org.apache.sqoop.cli.RelatedOptions {
public RelatedOptions() {
this("");
super();
}
public RelatedOptions(final String title) {
super();
this.relatedTitle = title;
}
public String getTitle() {
return relatedTitle;
}
@Override
public String toString() {
return relatedTitle + "\n" + super.toString();
super(title);
}
}

100
src/java/com/cloudera/sqoop/cli/SqoopParser.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -17,104 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloudera.sqoop.cli;
import java.util.ListIterator;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
/**
* Options parser that follows Sqoop's specific options-parsing needs.
* <ul>
* <li>Option values may contain the '&quot;' character as the first
* or last character in the value.</li>
* <li>The argument '--' must be preserved in the returned (unparsed)
* argument list.</li>
* </ul>
* @deprecated Moving to use org.apache.sqoop namespace.
*/
public class SqoopParser extends GnuParser {
public static final Log LOG = LogFactory.getLog(SqoopParser.class.getName());
// We need a handle to Option.addValueForProcessing(String).
// Since Hadoop will load this in a different classloader than
// this Sqoop class, we cannot see their package-specific methods.
// So we just call it by reflection. As long as we're at it, this
// allows us to also put SqoopParser in its own package.
private static java.lang.reflect.Method addValForProcessing;
static {
try {
addValForProcessing = Option.class.getDeclaredMethod(
"addValueForProcessing", String.class);
addValForProcessing.setAccessible(true);
} catch (NoSuchMethodException nsme) {
LOG.error("Could not load required method of Parser: "
+ StringUtils.stringifyException(nsme));
addValForProcessing = null;
}
}
@Override
/**
* Processes arguments to options but only strips matched quotes.
*/
public void processArgs(Option opt, ListIterator iter)
throws ParseException {
// Loop until an option is found.
while (iter.hasNext()) {
String str = (String) iter.next();
if (getOptions().hasOption(str) && str.startsWith("-")) {
// found an Option, not an argument.
iter.previous();
break;
}
// Otherwise, this is a value.
try {
// Note that we only strip matched quotes here.
addValForProcessing.invoke(opt, stripMatchedQuotes(str));
} catch (IllegalAccessException iae) {
throw new RuntimeException(iae);
} catch (java.lang.reflect.InvocationTargetException ite) {
// Any runtime exception thrown within addValForProcessing()
// will be wrapped in an InvocationTargetException.
iter.previous();
break;
} catch (RuntimeException re) {
iter.previous();
break;
}
}
if (opt.getValues() == null && !opt.hasOptionalArg()) {
throw new MissingArgumentException(opt);
}
}
/**
* Util.stripLeadingAndTrailingQuotes() will strip a '&quot;'
* character from either or both sides of a string. We only
* strip the matched pair.
*/
private String stripMatchedQuotes(String in) {
if (null == in || "\"".equals(in)) {
return in; // single quote stays as-is.
} else if(in.startsWith("\"") && in.endsWith("\"")) {
// Strip this pair of matched quotes.
return in.substring(1, in.length() - 1);
} else {
// return as-is.
return in;
}
}
public class SqoopParser
extends org.apache.sqoop.cli.SqoopParser {
}

132
src/java/com/cloudera/sqoop/cli/ToolOptions.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -20,134 +18,10 @@
package com.cloudera.sqoop.cli;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
/**
* Class that holds several sets of related options, providing a container
* for all the options associated with a single tool.
* The order in which sets of related options are added to this tool is
* preserved in printing and iteration.
* @deprecated Moving to use org.apache.sqoop namespace.
*/
public class ToolOptions implements Iterable<RelatedOptions> {
private List<RelatedOptions> optGroups;
public ToolOptions() {
this.optGroups = new ArrayList<RelatedOptions>();
}
/**
* Add a block of related options to the options for this tool.
* @param opts the set of RelatedOptions to add.
*/
public void addOptions(RelatedOptions opts) {
optGroups.add(opts);
}
/**
* Add a block of related options to the options for this tool,
* if a block has not already been added with the same title.
* @param opts the set of RelatedOptions to add.
*/
public void addUniqueOptions(RelatedOptions opts) {
if (!containsGroup(opts.getTitle())) {
optGroups.add(opts);
}
}
/**
* Reports whether this collection of RelatedOptions contains
* a RelatedOptions with the specified title.
* @param title the group title to search for
* @return true if a RelatedOptions with this group title is
* in the collection.
*/
public boolean containsGroup(String title) {
for (RelatedOptions related : this) {
if (related.getTitle().equals(title)) {
return true;
}
}
return false;
}
/**
* Provide an iterator over all sets of RelatedOptions.
* @return an iterator returning each RelatedOptions element.
*/
public Iterator<RelatedOptions> iterator() {
return optGroups.iterator();
}
/**
* Flatten the different sets of related options into a single collection
* of options.
* @return all options in the ToolOptions as a single set
*/
public Options merge() {
Options mergedOpts = new Options();
int totalOpts = 0;
for (RelatedOptions relatedOpts : this) {
for (Object optObj : relatedOpts.getOptions()) {
Option opt = (Option) optObj;
mergedOpts.addOption(opt);
totalOpts++;
}
}
return mergedOpts;
}
/**
* Print the help to the console using a default help formatter.
*/
public void printHelp() {
printHelp(new HelpFormatter());
}
/**
* Print the help to the console using the specified help formatter.
* @param formatter the HelpFormatter to use.
*/
public void printHelp(HelpFormatter formatter) {
printHelp(formatter, new PrintWriter(System.out, true));
}
/**
* Print the help to the specified PrintWriter, using the specified
* help formatter.
* @param formatter the HelpFormatter to use.
* @param pw the PrintWriter to emit to.
*/
public void printHelp(HelpFormatter formatter, PrintWriter pw) {
boolean first = true;
for (RelatedOptions optGroup : optGroups) {
if (!first) {
pw.println("");
}
pw.println(optGroup.getTitle() + ":");
formatter.printOptions(pw, formatter.getWidth(), optGroup, 0, 4);
first = false;
}
}
@Override
public String toString() {
StringWriter sw = new StringWriter();
printHelp(new HelpFormatter(), new PrintWriter(sw));
sw.flush();
return sw.getBuffer().toString();
}
public class ToolOptions
extends org.apache.sqoop.cli.ToolOptions {
}

71
src/java/com/cloudera/sqoop/config/ConfigurationConstants.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -17,70 +15,37 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloudera.sqoop.config;
/**
* Static constants that identify configuration keys, counter group names, and
* counter names.
* @deprecated Moving to use org.apache.sqoop namespace.
*/
public final class ConfigurationConstants {
/**
* The Configuration property identifying the current task id.
*/
public static final String PROP_MAPRED_TASK_ID = "mapred.task.id";
/**
* The Configuration property identifying the job's local directory.
*/
public static final String PROP_JOB_LOCAL_DIRECTORY = "job.local.dir";
/**
* The Configuration property identifying the number of map tasks to be used.
*/
public static final String PROP_MAPRED_MAP_TASKS = "mapred.map.tasks";
/**
* The Configuration property identifying the speculative execution flag for
* map tasks.
*/
public static final String PROP_MAPRED_TASK_ID =
org.apache.sqoop.config.ConfigurationConstants.PROP_MAPRED_TASK_ID;
public static final String PROP_JOB_LOCAL_DIRECTORY =
org.apache.sqoop.config.ConfigurationConstants.PROP_JOB_LOCAL_DIRECTORY;
public static final String PROP_MAPRED_MAP_TASKS =
org.apache.sqoop.config.ConfigurationConstants.PROP_MAPRED_MAP_TASKS;
public static final String PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC =
"mapred.map.tasks.speculative.execution";
/**
* The Configuration property identifying the speculative execution flag for
* reduce tasks.
*/
org.apache.sqoop.config.
ConfigurationConstants.PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC;
public static final String PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC =
"mapred.reduce.tasks.speculative.execution";
/**
* The Configuration property identifying the job tracker address.
*/
org.apache.sqoop.config.
ConfigurationConstants.PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC;
public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS =
"mapred.job.tracker";
/**
* The group name of task counters.
*/
org.apache.sqoop.config.
ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS;
public static final String COUNTER_GROUP_MAPRED_TASK_COUNTERS =
"org.apache.hadoop.mapred.Task$Counter";
/**
* The name of the counter that tracks output records from Map phase.
*/
org.apache.sqoop.config.
ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS;
public static final String COUNTER_MAP_OUTPUT_RECORDS =
"MAP_OUTPUT_RECORDS";
/**
* The name of the counter that tracks input records to the Map phase.
*/
org.apache.sqoop.config.ConfigurationConstants.COUNTER_MAP_OUTPUT_RECORDS;
public static final String COUNTER_MAP_INPUT_RECORDS =
"MAP_INPUT_RECORDS";
org.apache.sqoop.config.ConfigurationConstants.COUNTER_MAP_INPUT_RECORDS;
private ConfigurationConstants() {
// Disable Explicit Object Creation
}
}

121
src/java/com/cloudera/sqoop/config/ConfigurationHelper.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -17,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloudera.sqoop.config;
import java.io.IOException;
@ -25,149 +22,91 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.util.GenericOptionsParser;
import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
/**
* This class provides static helper methods that allow access and manipulation
* of job configuration. It is convenient to keep such access in one place in
* order to allow easy modifications when some of these aspects change from
* version to version of Hadoop.
* @deprecated Moving to use org.apache.sqoop namespace.
*/
public final class ConfigurationHelper {
/**
* Set the (hinted) number of map tasks for a job.
*/
public static void setJobNumMaps(Job job, int numMapTasks) {
job.getConfiguration().setInt(
ConfigurationConstants.PROP_MAPRED_MAP_TASKS, numMapTasks);
org.apache.sqoop.config.ConfigurationHelper.setJobNumMaps(job, numMapTasks);
}
/**
* Get the (hinted) number of map tasks for a job.
*/
public static int getJobNumMaps(JobContext job) {
return job.getConfiguration().getInt(
ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1);
return org.apache.sqoop.config.ConfigurationHelper.getJobNumMaps(job);
}
/**
* @return the number of mapper output records from a job using its counters.
*/
public static long getNumMapOutputRecords(Job job)
throws IOException, InterruptedException {
return job.getCounters().findCounter(
ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS,
ConfigurationConstants.COUNTER_MAP_OUTPUT_RECORDS).getValue();
return org.apache.sqoop.config.
ConfigurationHelper.getNumMapOutputRecords(job);
}
/**
* @return the number of mapper input records from a job using its counters.
*/
public static long getNumMapInputRecords(Job job)
throws IOException, InterruptedException {
return job.getCounters().findCounter(
ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS,
ConfigurationConstants.COUNTER_MAP_INPUT_RECORDS).getValue();
return org.apache.sqoop.config.
ConfigurationHelper.getNumMapInputRecords(job);
}
/**
* Get the (hinted) number of map tasks for a job.
*/
public static int getConfNumMaps(Configuration conf) {
return conf.getInt(ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1);
return org.apache.sqoop.config.ConfigurationHelper.getConfNumMaps(conf);
}
/**
* Set the mapper speculative execution property for a job.
*/
public static void setJobMapSpeculativeExecution(Job job, boolean isEnabled) {
job.getConfiguration().setBoolean(
ConfigurationConstants.PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC,
isEnabled);
org.apache.sqoop.config.
ConfigurationHelper.setJobMapSpeculativeExecution(job, isEnabled);
}
/**
* Set the reducer speculative execution property for a job.
*/
public static void setJobReduceSpeculativeExecution(
Job job, boolean isEnabled) {
job.getConfiguration().setBoolean(
ConfigurationConstants.PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC,
isEnabled);
org.apache.sqoop.config.
ConfigurationHelper.setJobReduceSpeculativeExecution(job, isEnabled);
}
/**
* Sets the Jobtracker address to use for a job.
*/
public static void setJobtrackerAddr(Configuration conf, String addr) {
conf.set(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS, addr);
org.apache.sqoop.config.
ConfigurationHelper.setJobtrackerAddr(conf, addr);
}
/**
* @return the Configuration property identifying a DBWritable to use.
*/
public static String getDbInputClassProperty() {
return DBConfiguration.INPUT_CLASS_PROPERTY;
return org.apache.sqoop.config.
ConfigurationHelper.getDbInputClassProperty();
}
/**
* @return the Configuration property identifying the DB username.
*/
public static String getDbUsernameProperty() {
return DBConfiguration.USERNAME_PROPERTY;
return org.apache.sqoop.config.
ConfigurationHelper.getDbUsernameProperty();
}
/**
* @return the Configuration property identifying the DB password.
*/
public static String getDbPasswordProperty() {
return DBConfiguration.PASSWORD_PROPERTY;
return org.apache.sqoop.config.
ConfigurationHelper.getDbPasswordProperty();
}
/**
* @return the Configuration property identifying the DB connect string.
*/
public static String getDbUrlProperty() {
return DBConfiguration.URL_PROPERTY;
return org.apache.sqoop.config.
ConfigurationHelper.getDbUrlProperty();
}
/**
* @return the Configuration property identifying the DB input table.
*/
public static String getDbInputTableNameProperty() {
return DBConfiguration.INPUT_TABLE_NAME_PROPERTY;
return org.apache.sqoop.config.
ConfigurationHelper.getDbInputTableNameProperty();
}
/**
* @return the Configuration property specifying WHERE conditions for the
* db table.
*/
public static String getDbInputConditionsProperty() {
return DBConfiguration.INPUT_CONDITIONS_PROPERTY;
return org.apache.sqoop.config.
ConfigurationHelper.getDbInputConditionsProperty();
}
/**
* Parse arguments in 'args' via the GenericOptionsParser and
* embed the results in the supplied configuration.
* @param conf the configuration to populate with generic options.
* @param args the arguments to process.
* @return the unused args to be passed to the application itself.
*/
public static String [] parseGenericOptions(
Configuration conf, String [] args) throws IOException {
// This needs to be shimmed because in Apache Hadoop this can throw
// an IOException, but it does not do so in CDH. We just mandate in
// this method that an IOException is possible.
GenericOptionsParser genericParser = new GenericOptionsParser(
conf, args);
return genericParser.getRemainingArgs();
return org.apache.sqoop.config.
ConfigurationHelper.parseGenericOptions(conf, args);
}
private ConfigurationHelper() {
// Disable explicit object creation
}
}

49
src/java/org/apache/sqoop/cli/RelatedOptions.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.cli;
import org.apache.commons.cli.Options;
/**
* Class that extends a set of options to provide a title for a
* related set of options.
*/
public class RelatedOptions extends Options {
private String relatedTitle;
public RelatedOptions() {
this("");
}
public RelatedOptions(final String title) {
super();
this.relatedTitle = title;
}
public String getTitle() {
return relatedTitle;
}
@Override
public String toString() {
return relatedTitle + "\n" + super.toString();
}
}

118
src/java/org/apache/sqoop/cli/SqoopParser.java Normal file
View File

@ -0,0 +1,118 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.cli;
import java.util.ListIterator;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
/**
* Options parser that follows Sqoop's specific options-parsing needs.
* <ul>
* <li>Option values may contain the '&quot;' character as the first
* or last character in the value.</li>
* <li>The argument '--' must be preserved in the returned (unparsed)
* argument list.</li>
* </ul>
*/
public class SqoopParser extends GnuParser {
public static final Log LOG = LogFactory.getLog(SqoopParser.class.getName());
// We need a handle to Option.addValueForProcessing(String).
// Since Hadoop will load this in a different classloader than
// this Sqoop class, we cannot see their package-specific methods.
// So we just call it by reflection. As long as we're at it, this
// allows us to also put SqoopParser in its own package.
private static java.lang.reflect.Method addValForProcessing;
static {
try {
addValForProcessing = Option.class.getDeclaredMethod(
"addValueForProcessing", String.class);
addValForProcessing.setAccessible(true);
} catch (NoSuchMethodException nsme) {
LOG.error("Could not load required method of Parser: "
+ StringUtils.stringifyException(nsme));
addValForProcessing = null;
}
}
@Override
/**
* Processes arguments to options but only strips matched quotes.
*/
public void processArgs(Option opt, ListIterator iter)
throws ParseException {
// Loop until an option is found.
while (iter.hasNext()) {
String str = (String) iter.next();
if (getOptions().hasOption(str) && str.startsWith("-")) {
// found an Option, not an argument.
iter.previous();
break;
}
// Otherwise, this is a value.
try {
// Note that we only strip matched quotes here.
addValForProcessing.invoke(opt, stripMatchedQuotes(str));
} catch (IllegalAccessException iae) {
throw new RuntimeException(iae);
} catch (java.lang.reflect.InvocationTargetException ite) {
// Any runtime exception thrown within addValForProcessing()
// will be wrapped in an InvocationTargetException.
iter.previous();
break;
} catch (RuntimeException re) {
iter.previous();
break;
}
}
if (opt.getValues() == null && !opt.hasOptionalArg()) {
throw new MissingArgumentException(opt);
}
}
/**
* Util.stripLeadingAndTrailingQuotes() will strip a '&quot;'
* character from either or both sides of a string. We only
* strip the matched pair.
*/
private String stripMatchedQuotes(String in) {
if (null == in || "\"".equals(in)) {
return in; // single quote stays as-is.
} else if(in.startsWith("\"") && in.endsWith("\"")) {
// Strip this pair of matched quotes.
return in.substring(1, in.length() - 1);
} else {
// return as-is.
return in;
}
}
}

153
src/java/org/apache/sqoop/cli/ToolOptions.java Normal file
View File

@ -0,0 +1,153 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.cli;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import com.cloudera.sqoop.cli.RelatedOptions;
/**
* Class that holds several sets of related options, providing a container
* for all the options associated with a single tool.
* The order in which sets of related options are added to this tool is
* preserved in printing and iteration.
*/
public class ToolOptions implements Iterable<RelatedOptions> {
private List<RelatedOptions> optGroups;
public ToolOptions() {
this.optGroups = new ArrayList<RelatedOptions>();
}
/**
* Add a block of related options to the options for this tool.
* @param opts the set of RelatedOptions to add.
*/
public void addOptions(RelatedOptions opts) {
optGroups.add(opts);
}
/**
* Add a block of related options to the options for this tool,
* if a block has not already been added with the same title.
* @param opts the set of RelatedOptions to add.
*/
public void addUniqueOptions(RelatedOptions opts) {
if (!containsGroup(opts.getTitle())) {
optGroups.add(opts);
}
}
/**
* Reports whether this collection of RelatedOptions contains
* a RelatedOptions with the specified title.
* @param title the group title to search for
* @return true if a RelatedOptions with this group title is
* in the collection.
*/
public boolean containsGroup(String title) {
for (RelatedOptions related : this) {
if (related.getTitle().equals(title)) {
return true;
}
}
return false;
}
/**
* Provide an iterator over all sets of RelatedOptions.
* @return an iterator returning each RelatedOptions element.
*/
public Iterator<RelatedOptions> iterator() {
return optGroups.iterator();
}
/**
* Flatten the different sets of related options into a single collection
* of options.
* @return all options in the ToolOptions as a single set
*/
public Options merge() {
Options mergedOpts = new Options();
int totalOpts = 0;
for (RelatedOptions relatedOpts : this) {
for (Object optObj : relatedOpts.getOptions()) {
Option opt = (Option) optObj;
mergedOpts.addOption(opt);
totalOpts++;
}
}
return mergedOpts;
}
/**
* Print the help to the console using a default help formatter.
*/
public void printHelp() {
printHelp(new HelpFormatter());
}
/**
* Print the help to the console using the specified help formatter.
* @param formatter the HelpFormatter to use.
*/
public void printHelp(HelpFormatter formatter) {
printHelp(formatter, new PrintWriter(System.out, true));
}
/**
* Print the help to the specified PrintWriter, using the specified
* help formatter.
* @param formatter the HelpFormatter to use.
* @param pw the PrintWriter to emit to.
*/
public void printHelp(HelpFormatter formatter, PrintWriter pw) {
boolean first = true;
for (RelatedOptions optGroup : optGroups) {
if (!first) {
pw.println("");
}
pw.println(optGroup.getTitle() + ":");
formatter.printOptions(pw, formatter.getWidth(), optGroup, 0, 4);
first = false;
}
}
@Override
public String toString() {
StringWriter sw = new StringWriter();
printHelp(new HelpFormatter(), new PrintWriter(sw));
sw.flush();
return sw.getBuffer().toString();
}
}

84
src/java/org/apache/sqoop/config/ConfigurationConstants.java Normal file
View File

@ -0,0 +1,84 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.config;
/**
* Static constants that identify configuration keys, counter group names, and
* counter names.
*/
public final class ConfigurationConstants {
/**
* The Configuration property identifying the current task id.
*/
public static final String PROP_MAPRED_TASK_ID = "mapred.task.id";
/**
* The Configuration property identifying the job's local directory.
*/
public static final String PROP_JOB_LOCAL_DIRECTORY = "job.local.dir";
/**
* The Configuration property identifying the number of map tasks to be used.
*/
public static final String PROP_MAPRED_MAP_TASKS = "mapred.map.tasks";
/**
* The Configuration property identifying the speculative execution flag for
* map tasks.
*/
public static final String PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC =
"mapred.map.tasks.speculative.execution";
/**
* The Configuration property identifying the speculative execution flag for
* reduce tasks.
*/
public static final String PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC =
"mapred.reduce.tasks.speculative.execution";
/**
* The Configuration property identifying the job tracker address.
*/
public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS =
"mapred.job.tracker";
/**
* The group name of task counters.
*/
public static final String COUNTER_GROUP_MAPRED_TASK_COUNTERS =
"org.apache.hadoop.mapred.Task$Counter";
/**
* The name of the counter that tracks output records from Map phase.
*/
public static final String COUNTER_MAP_OUTPUT_RECORDS =
"MAP_OUTPUT_RECORDS";
/**
* The name of the counter that tracks input records to the Map phase.
*/
public static final String COUNTER_MAP_INPUT_RECORDS =
"MAP_INPUT_RECORDS";
private ConfigurationConstants() {
// Disable Explicit Object Creation
}
}

171
src/java/org/apache/sqoop/config/ConfigurationHelper.java Normal file
View File

@ -0,0 +1,171 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.config;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.util.GenericOptionsParser;
import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
/**
* This class provides static helper methods that allow access and manipulation
* of job configuration. It is convenient to keep such access in one place in
* order to allow easy modifications when some of these aspects change from
* version to version of Hadoop.
*/
public final class ConfigurationHelper {
/**
* Set the (hinted) number of map tasks for a job.
*/
public static void setJobNumMaps(Job job, int numMapTasks) {
job.getConfiguration().setInt(
ConfigurationConstants.PROP_MAPRED_MAP_TASKS, numMapTasks);
}
/**
* Get the (hinted) number of map tasks for a job.
*/
public static int getJobNumMaps(JobContext job) {
return job.getConfiguration().getInt(
ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1);
}
/**
* @return the number of mapper output records from a job using its counters.
*/
public static long getNumMapOutputRecords(Job job)
throws IOException, InterruptedException {
return job.getCounters().findCounter(
ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS,
ConfigurationConstants.COUNTER_MAP_OUTPUT_RECORDS).getValue();
}
/**
* @return the number of mapper input records from a job using its counters.
*/
public static long getNumMapInputRecords(Job job)
throws IOException, InterruptedException {
return job.getCounters().findCounter(
ConfigurationConstants.COUNTER_GROUP_MAPRED_TASK_COUNTERS,
ConfigurationConstants.COUNTER_MAP_INPUT_RECORDS).getValue();
}
/**
* Get the (hinted) number of map tasks for a job.
*/
public static int getConfNumMaps(Configuration conf) {
return conf.getInt(ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1);
}
/**
* Set the mapper speculative execution property for a job.
*/
public static void setJobMapSpeculativeExecution(Job job, boolean isEnabled) {
job.getConfiguration().setBoolean(
ConfigurationConstants.PROP_MAPRED_MAP_TASKS_SPECULATIVE_EXEC,
isEnabled);
}
/**
* Set the reducer speculative execution property for a job.
*/
public static void setJobReduceSpeculativeExecution(
Job job, boolean isEnabled) {
job.getConfiguration().setBoolean(
ConfigurationConstants.PROP_MAPRED_REDUCE_TASKS_SPECULATIVE_EXEC,
isEnabled);
}
/**
* Sets the Jobtracker address to use for a job.
*/
public static void setJobtrackerAddr(Configuration conf, String addr) {
conf.set(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS, addr);
}
/**
* @return the Configuration property identifying a DBWritable to use.
*/
public static String getDbInputClassProperty() {
return DBConfiguration.INPUT_CLASS_PROPERTY;
}
/**
* @return the Configuration property identifying the DB username.
*/
public static String getDbUsernameProperty() {
return DBConfiguration.USERNAME_PROPERTY;
}
/**
* @return the Configuration property identifying the DB password.
*/
public static String getDbPasswordProperty() {
return DBConfiguration.PASSWORD_PROPERTY;
}
/**
* @return the Configuration property identifying the DB connect string.
*/
public static String getDbUrlProperty() {
return DBConfiguration.URL_PROPERTY;
}
/**
* @return the Configuration property identifying the DB input table.
*/
public static String getDbInputTableNameProperty() {
return DBConfiguration.INPUT_TABLE_NAME_PROPERTY;
}
/**
* @return the Configuration property specifying WHERE conditions for the
* db table.
*/
public static String getDbInputConditionsProperty() {
return DBConfiguration.INPUT_CONDITIONS_PROPERTY;
}
/**
* Parse arguments in 'args' via the GenericOptionsParser and
* embed the results in the supplied configuration.
* @param conf the configuration to populate with generic options.
* @param args the arguments to process.
* @return the unused args to be passed to the application itself.
*/
public static String [] parseGenericOptions(
Configuration conf, String [] args) throws IOException {
// This needs to be shimmed because in Apache Hadoop this can throw
// an IOException, but it does not do so in CDH. We just mandate in
// this method that an IOException is possible.
GenericOptionsParser genericParser = new GenericOptionsParser(
conf, args);
return genericParser.getRemainingArgs();
}
private ConfigurationHelper() {
// Disable explicit object creation
}
}

22
src/test/com/cloudera/sqoop/manager/PostgresqlTest.java
View File

@ -178,7 +178,8 @@ public void setUpData(String tableName) {
}
private String [] getArgv(boolean isDirect, String tableName) {
private String [] getArgv(boolean isDirect, String tableName,
String... extraArgs) {
ArrayList<String> args = new ArrayList<String>();
CommonArgs.addHadoopFlags(args);
@ -198,11 +199,15 @@ public void setUpData(String tableName) {
args.add("--direct");
}
for (String arg : extraArgs) {
args.add(arg);
}
return args.toArray(new String[0]);
}
private void doImportAndVerify(boolean isDirect, String [] expectedResults,
String tableName) throws IOException {
String tableName, String... extraArgs) throws IOException {
Path warehousePath = new Path(this.getWarehouseDir());
Path tablePath = new Path(warehousePath, tableName);
@ -220,7 +225,7 @@ private void doImportAndVerify(boolean isDirect, String [] expectedResults,
FileListing.recursiveDeleteDir(tableFile);
}
String [] argv = getArgv(isDirect, tableName);
String [] argv = getArgv(isDirect, tableName, extraArgs);
try {
runImport(argv);
} catch (IOException ioe) {
@ -289,4 +294,15 @@ public void testTableNameWithSpecialCharacter() throws IOException {
doImportAndVerify(false, expectedResults, SPECIAL_TABLE_NAME);
}
@Test
public void testIncrementalImport() throws IOException {
String [] expectedResults = { };
String [] extraArgs = { "--incremental", "lastmodified",
"--check-column", "start_date",
};
doImportAndVerify(false, expectedResults, TABLE_NAME, extraArgs);
}
}