github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 18:00:26 +08:00
Code Issues Projects Releases Wiki Activity

Add checkstyle to build process.

Browse Source 
Fix the overwhelming majority of existing violations.

From: Aaron Kimball <aaron@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149899 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Bayer 2011-07-22 20:03:47 +00:00
parent e926bf1fe0
commit 0b96b5f1c3
136 changed files with 1981 additions and 1275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
build.xml
View File

@ -88,6 +88,12 @@
<property name="cobertura.format" value="html" /> <!-- may be 'xml' -->
<property name="cobertura.class.dir" value="${cobertura.dir}/classes" />
<!-- Checking code style -->
<property name="checkstyle.xml" value="${test.dir}/checkstyle.xml" />
<property name="checkstyle.format.xsl"
value="${test.dir}/checkstyle-noframes.xsl" />
<property name="checkstyle.report.dir" value="${build.dir}" />
<!-- When testing with non-free JDBC drivers, override this parameter
to contain the path to the driver library dir.
@ -599,6 +605,24 @@
file="${cobertura.home}/cobertura.jar" />
</target>
<target name="checkstyle" depends="ivy-retrieve-checkstyle"
description="Check source code conventions">
<taskdef resource="checkstyletask.properties">
<classpath refid="${name}.checkstyle.classpath" />
</taskdef>
<mkdir dir="${checkstyle.report.dir}" />
<checkstyle config="${checkstyle.xml}" failOnViolation="false">
<fileset dir="${base.src.dir}" includes="**/*.java" />
<formatter type="xml"
toFile="${checkstyle.report.dir}/checkstyle-errors.xml" />
</checkstyle>
<xslt style="${checkstyle.format.xsl}"
in="${checkstyle.report.dir}/checkstyle-errors.xml"
out="${checkstyle.report.dir}/checkstyle-errors.html" />
</target>
<target name="ivy-probe-antlib" >
<condition property="ivy.found">
<typefound uri="antlib:org.apache.ivy.ant" name="cleancache"/>
@ -690,4 +714,14 @@
conf="${hadoop.dist}test" />
</target>
<!-- retrieve ivy-managed artifacts for checkstyle -->
<target name="ivy-resolve-checkstyle" depends="ivy-init">
<ivy:resolve settingsRef="${name}.ivy.settings" conf="checkstyle" />
</target>
<target name="ivy-retrieve-checkstyle" depends="ivy-resolve-checkstyle">
<ivy:retrieve settingsRef="${name}.ivy.settings"
pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}" sync="true" />
<ivy:cachepath pathid="${name}.checkstyle.classpath" conf="checkstyle" />
</target>
</project>

7
ivy.xml
View File

@ -52,6 +52,9 @@
Things we actually need to ship are marked with a 'redist'
configuration. -->
<conf name="redist" visibility="private" extends="master" />
<!-- configuration for running checkstyle -->
<conf name="checkstyle" visibility="private" />
</configurations>
<publications>
@ -92,5 +95,9 @@
conf="common->default"/>
<dependency org="commons-io" name="commons-io" rev="${commons-io.version}"
conf="common->default;redist->default"/>
<!-- dependencies for static analysis -->
<dependency org="checkstyle" name="checkstyle" rev="${checkstyle.version}"
conf="checkstyle->default" />
</dependencies>
</ivy-module>

2
ivy/libraries.properties
View File

@ -16,6 +16,8 @@
# This properties file lists the versions of the various artifacts we use.
# It drives ivy and the generation of a maven POM
checkstyle.version=5.0
commons-cli.version=1.2
commons-io.version=1.4
commons-logging.version=1.0.4

18
src/java/org/apache/hadoop/sqoop/ConnFactory.java
View File

@ -37,7 +37,8 @@
*
* This class delegates the actual responsibility for instantiating
* ConnManagers to one or more instances of ManagerFactory. ManagerFactories
* are consulted in the order specified in sqoop-site.xml (sqoop.connection.factories).
* are consulted in the order specified in sqoop-site.xml
* (sqoop.connection.factories).
*/
public class ConnFactory {
@ -51,10 +52,13 @@ public ConnFactory(Configuration conf) {
/** The sqoop-site.xml configuration property used to set the list of
* available ManagerFactories.
*/
public final static String FACTORY_CLASS_NAMES_KEY = "sqoop.connection.factories";
public static final String FACTORY_CLASS_NAMES_KEY =
"sqoop.connection.factories";
// The default value for sqoop.connection.factories is the name of the DefaultManagerFactory.
final static String DEFAULT_FACTORY_CLASS_NAMES = DefaultManagerFactory.class.getName();
// The default value for sqoop.connection.factories is the
// name of the DefaultManagerFactory.
static final String DEFAULT_FACTORY_CLASS_NAMES =
DefaultManagerFactory.class.getName();
/** The list of ManagerFactory instances consulted by getManager().
*/
@ -76,7 +80,8 @@ private void instantiateFactories(Configuration conf) {
LOG.debug("Loaded manager factory: " + className);
factories.add(factory);
} catch (ClassNotFoundException cnfe) {
LOG.error("Could not load ManagerFactory " + className + " (not found)");
LOG.error("Could not load ManagerFactory " + className
+ " (not found)");
}
}
}
@ -98,7 +103,8 @@ public ConnManager getManager(SqoopOptions opts) throws IOException {
}
}
throw new IOException("No manager for connect string: " + opts.getConnectString());
throw new IOException("No manager for connect string: "
+ opts.getConnectString());
}
}

18
src/java/org/apache/hadoop/sqoop/Sqoop.java
View File

@ -18,11 +18,7 @@
package org.apache.hadoop.sqoop;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
@ -32,18 +28,8 @@
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.sqoop.cli.ToolOptions;
import org.apache.hadoop.sqoop.hive.HiveImport;
import org.apache.hadoop.sqoop.manager.ConnManager;
import org.apache.hadoop.sqoop.manager.ExportJobContext;
import org.apache.hadoop.sqoop.manager.ImportJobContext;
import org.apache.hadoop.sqoop.orm.ClassWriter;
import org.apache.hadoop.sqoop.orm.CompilationManager;
import org.apache.hadoop.sqoop.shims.ShimLoader;
import org.apache.hadoop.sqoop.tool.HelpTool;
import org.apache.hadoop.sqoop.tool.SqoopTool;
import org.apache.hadoop.sqoop.util.ExportException;
import org.apache.hadoop.sqoop.util.ImportException;
/**
* Main entry-point for Sqoop
@ -201,7 +187,8 @@ public static int runTool(String [] args) {
String toolName = args[0];
SqoopTool tool = SqoopTool.getTool(toolName);
if (null == tool) {
System.err.println("No such sqoop tool: " + toolName + ". See 'sqoop help'.");
System.err.println("No such sqoop tool: " + toolName
+ ". See 'sqoop help'.");
return 1;
}
@ -219,3 +206,4 @@ public static void main(String [] args) {
System.exit(ret);
}
}

171
src/java/org/apache/hadoop/sqoop/SqoopOptions.java
View File

@ -37,14 +37,14 @@
import org.apache.log4j.Logger;
/**
* Command-line arguments used by Sqoop
* Command-line arguments used by Sqoop.
*/
public class SqoopOptions {
public static final Log LOG = LogFactory.getLog(SqoopOptions.class.getName());
/**
* Thrown when invalid cmdline options are given
* Thrown when invalid cmdline options are given.
*/
@SuppressWarnings("serial")
public static class InvalidOptionsException extends Exception {
@ -64,7 +64,7 @@ public String toString() {
}
}
// selects in-HDFS destination file format
/** Selects in-HDFS destination file format. */
public enum FileLayout {
TextFile,
SequenceFile
@ -98,18 +98,27 @@ public enum FileLayout {
private boolean overwriteHiveTable;
private String hiveTableName;
private String packageName; // package to prepend to auto-named classes.
private String className; // package+class to apply to individual table import.
// also used as an *input* class with existingJarFile.
private String existingJarFile; // Name of a jar containing existing table definition
// class to use.
// package+class to apply to individual table import.
// also used as an *input* class with existingJarFile.
private String className;
// Name of a jar containing existing table definition
// class to use.
private String existingJarFile;
private int numMappers;
private boolean useCompression;
private long directSplitSize; // In direct mode, open a new stream every X bytes.
private long maxInlineLobSize; // Max size of an inline LOB; larger LOBs are written
// to external files on disk.
// In direct mode, open a new stream every X bytes.
private long directSplitSize;
private String exportDir; // HDFS path to read from when performing an export
// Max size of an inline LOB; larger LOBs are written
// to external files on disk.
private long maxInlineLobSize;
// HDFS path to read from when performing an export
private String exportDir;
private char inputFieldDelim;
private char inputRecordDelim;
@ -142,7 +151,7 @@ public SqoopOptions(Configuration conf) {
}
/**
* Alternate SqoopOptions interface used mostly for unit testing
* Alternate SqoopOptions interface used mostly for unit testing.
* @param connect JDBC connect string to use
* @param table Table to read
*/
@ -153,19 +162,22 @@ public SqoopOptions(final String connect, final String table) {
this.tableName = table;
}
private boolean getBooleanProperty(Properties props, String propName, boolean defaultValue) {
private boolean getBooleanProperty(Properties props, String propName,
boolean defaultValue) {
String str = props.getProperty(propName,
Boolean.toString(defaultValue)).toLowerCase();
return "true".equals(str) || "yes".equals(str) || "1".equals(str);
}
private long getLongProperty(Properties props, String propName, long defaultValue) {
private long getLongProperty(Properties props, String propName,
long defaultValue) {
String str = props.getProperty(propName,
Long.toString(defaultValue)).toLowerCase();
try {
return Long.parseLong(str);
} catch (NumberFormatException nfe) {
LOG.warn("Could not parse integer value for config parameter " + propName);
LOG.warn("Could not parse integer value for config parameter "
+ propName);
return defaultValue;
}
}
@ -189,31 +201,40 @@ private void loadFromProperties() {
this.username = props.getProperty("db.username", this.username);
this.password = props.getProperty("db.password", this.password);
this.tableName = props.getProperty("db.table", this.tableName);
this.connectString = props.getProperty("db.connect.url", this.connectString);
this.connectString = props.getProperty("db.connect.url",
this.connectString);
this.splitByCol = props.getProperty("db.split.column", this.splitByCol);
this.whereClause = props.getProperty("db.where.clause", this.whereClause);
this.driverClassName = props.getProperty("jdbc.driver", this.driverClassName);
this.warehouseDir = props.getProperty("hdfs.warehouse.dir", this.warehouseDir);
this.driverClassName = props.getProperty("jdbc.driver",
this.driverClassName);
this.warehouseDir = props.getProperty("hdfs.warehouse.dir",
this.warehouseDir);
this.hiveHome = props.getProperty("hive.home", this.hiveHome);
this.className = props.getProperty("java.classname", this.className);
this.packageName = props.getProperty("java.packagename", this.packageName);
this.existingJarFile = props.getProperty("java.jar.file", this.existingJarFile);
this.packageName = props.getProperty("java.packagename",
this.packageName);
this.existingJarFile = props.getProperty("java.jar.file",
this.existingJarFile);
this.exportDir = props.getProperty("export.dir", this.exportDir);
this.direct = getBooleanProperty(props, "direct.import", this.direct);
this.hiveImport = getBooleanProperty(props, "hive.import", this.hiveImport);
this.overwriteHiveTable = getBooleanProperty(props, "hive.overwrite.table", this.overwriteHiveTable);
this.useCompression = getBooleanProperty(props, "compression", this.useCompression);
this.hiveImport = getBooleanProperty(props, "hive.import",
this.hiveImport);
this.overwriteHiveTable = getBooleanProperty(props,
"hive.overwrite.table", this.overwriteHiveTable);
this.useCompression = getBooleanProperty(props, "compression",
this.useCompression);
this.directSplitSize = getLongProperty(props, "direct.split.size",
this.directSplitSize);
} catch (IOException ioe) {
LOG.error("Could not read properties file " + DEFAULT_CONFIG_FILE + ": " + ioe.toString());
LOG.error("Could not read properties file " + DEFAULT_CONFIG_FILE + ": "
+ ioe.toString());
} finally {
if (null != istream) {
try {
istream.close();
} catch (IOException ioe) {
// ignore this; we're closing.
// Ignore this; we're closing.
}
}
}
@ -221,7 +242,7 @@ private void loadFromProperties() {
/**
* @return the temp directory to use; this is guaranteed to end with
* the file separator character (e.g., '/')
* the file separator character (e.g., '/').
*/
public String getTempDir() {
return this.tmpDir;
@ -280,17 +301,19 @@ private void initDefaults(Configuration baseConfiguration) {
}
/**
* Given a string containing a single character or an escape sequence representing
* a char, return that char itself.
* Given a string containing a single character or an escape sequence
* representing a char, return that char itself.
*
* Normal literal characters return themselves: "x" -&gt; 'x', etc.
* Strings containing a '\' followed by one of t, r, n, or b escape to the usual
* character as seen in Java: "\n" -&gt; (newline), etc.
* Strings containing a '\' followed by one of t, r, n, or b escape to the
* usual character as seen in Java: "\n" -&gt; (newline), etc.
*
* Strings like "\0ooo" return the character specified by the octal sequence 'ooo'
* Strings like "\0xhhh" or "\0Xhhh" return the character specified by the hex sequence 'hhh'
* Strings like "\0ooo" return the character specified by the octal sequence
* 'ooo'. Strings like "\0xhhh" or "\0Xhhh" return the character specified by
* the hex sequence 'hhh'.
*
* If the input string contains leading or trailing spaces, these are ignored.
* If the input string contains leading or trailing spaces, these are
* ignored.
*/
public static char toChar(String charish) throws InvalidOptionsException {
if (null == charish || charish.length() == 0) {
@ -300,8 +323,9 @@ public static char toChar(String charish) throws InvalidOptionsException {
if (charish.startsWith("\\0x") || charish.startsWith("\\0X")) {
if (charish.length() == 3) {
throw new InvalidOptionsException("Base-16 value expected for character argument."
+ "\nTry --help for usage instructions.");
throw new InvalidOptionsException(
"Base-16 value expected for character argument."
+ "\nTry --help for usage instructions.");
} else {
String valStr = charish.substring(3);
int val = Integer.parseInt(valStr, 16);
@ -323,7 +347,8 @@ public static char toChar(String charish) throws InvalidOptionsException {
return '\\';
} else if (charish.length() > 2) {
// we don't have any 3+ char escape strings.
throw new InvalidOptionsException("Cannot understand character argument: " + charish
throw new InvalidOptionsException(
"Cannot understand character argument: " + charish
+ "\nTry --help for usage instructions.");
} else {
// this is some sort of normal 1-character escape sequence.
@ -344,7 +369,8 @@ public static char toChar(String charish) throws InvalidOptionsException {
case '\\':
return '\\';
default:
throw new InvalidOptionsException("Cannot understand character argument: " + charish
throw new InvalidOptionsException(
"Cannot understand character argument: " + charish
+ "\nTry --help for usage instructions.");
}
}
@ -359,8 +385,9 @@ public static char toChar(String charish) throws InvalidOptionsException {
}
}
/** get the temporary directory; guaranteed to end in File.separator
* (e.g., '/')
/**
* Get the temporary directory; guaranteed to end in File.separator
* (e.g., '/').
*/
public String getTmpDir() {
return tmpDir;
@ -447,7 +474,8 @@ public String getPassword() {
}
/**
* Allow the user to enter his password on the console without printing characters.
* Allow the user to enter his password on the console without printing
* characters.
* @return the password as a string
*/
private String securePasswordEntry() {
@ -475,7 +503,7 @@ public void setDirectMode(boolean isDirect) {
}
/**
* @return the number of map tasks to use for import
* @return the number of map tasks to use for import.
*/
public int getNumMappers() {
return this.numMappers;
@ -486,7 +514,7 @@ public void setNumMappers(int numMappers) {
}
/**
* @return the user-specified absolute class name for the table
* @return the user-specified absolute class name for the table.
*/
public String getClassName() {
return className;
@ -497,7 +525,8 @@ public void setClassName(String className) {
}
/**
* @return the user-specified package to prepend to table names via --package-name.
* @return the user-specified package to prepend to table names via
* --package-name.
*/
public String getPackageName() {
return packageName;
@ -515,7 +544,7 @@ public void setHiveHome(String hiveHome) {
this.hiveHome = hiveHome;
}
/** @return true if we should import the table into Hive */
/** @return true if we should import the table into Hive. */
public boolean doHiveImport() {
return hiveImport;
}
@ -525,7 +554,7 @@ public void setHiveImport(boolean hiveImport) {
}
/**
* @return the user-specified option to overwrite existing table in hive
* @return the user-specified option to overwrite existing table in hive.
*/
public boolean doOverwriteHiveTable() {
return overwriteHiveTable;
@ -536,7 +565,7 @@ public void setOverwriteHiveTable(boolean overwrite) {
}
/**
* @return location where .java files go; guaranteed to end with '/'
* @return location where .java files go; guaranteed to end with '/'.
*/
public String getCodeOutputDir() {
if (codeOutputDir.endsWith(File.separator)) {
@ -551,7 +580,8 @@ public void setCodeOutputDir(String outputDir) {
}
/**
* @return location where .jar and .class files go; guaranteed to end with '/'
* @return location where .jar and .class files go; guaranteed to end with
* '/'.
*/
public String getJarOutputDir() {
if (jarOutputDir.endsWith(File.separator)) {
@ -566,7 +596,7 @@ public void setJarOutputDir(String outDir) {
}
/**
* Return the value of $HADOOP_HOME
* Return the value of $HADOOP_HOME.
* @return $HADOOP_HOME, or null if it's not set.
*/
public String getHadoopHome() {
@ -589,7 +619,7 @@ public void setDebugSqlCmd(String sqlStatement) {
}
/**
* @return The JDBC driver class name specified with --driver
* @return The JDBC driver class name specified with --driver.
*/
public String getDriverClassName() {
return driverClassName;
@ -622,8 +652,8 @@ public void setFileLayout(FileLayout layout) {
}
/**
* @return the field delimiter to use when parsing lines. Defaults to the field delim
* to use when printing lines
* @return the field delimiter to use when parsing lines. Defaults to the
* field delim to use when printing lines.
*/
public char getInputFieldDelim() {
if (inputFieldDelim == '\000') {
@ -638,8 +668,8 @@ public void setInputFieldsTerminatedBy(char c) {
}
/**
* @return the record delimiter to use when parsing lines. Defaults to the record delim
* to use when printing lines.
* @return the record delimiter to use when parsing lines. Defaults to the
* record delim to use when printing lines.
*/
public char getInputRecordDelim() {
if (inputRecordDelim == '\000') {
@ -654,8 +684,8 @@ public void setInputLinesTerminatedBy(char c) {
}
/**
* @return the character that may enclose fields when parsing lines. Defaults to the
* enclosing-char to use when printing lines.
* @return the character that may enclose fields when parsing lines.
* Defaults to the enclosing-char to use when printing lines.
*/
public char getInputEnclosedBy() {
if (inputEnclosedBy == '\000') {
@ -670,8 +700,8 @@ public void setInputEnclosedBy(char c) {
}
/**
* @return the escape character to use when parsing lines. Defaults to the escape
* character used when printing lines.
* @return the escape character to use when parsing lines. Defaults to the
* escape character used when printing lines.
*/
public char getInputEscapedBy() {
if (inputEscapedBy == '\000') {
@ -686,8 +716,9 @@ public void setInputEscapedBy(char c) {
}
/**
* @return true if fields must be enclosed by the --enclosed-by character when parsing.
* Defaults to false. Set true when --input-enclosed-by is used.
* @return true if fields must be enclosed by the --enclosed-by character
* when parsing. Defaults to false. Set true when --input-enclosed-by is
* used.
*/
public boolean isInputEncloseRequired() {
if (inputEnclosedBy == '\000') {
@ -702,7 +733,8 @@ public void setInputEncloseRequired(boolean required) {
}
/**
* @return the character to print between fields when importing them to text.
* @return the character to print between fields when importing them to
* text.
*/
public char getOutputFieldDelim() {
return this.outputFieldDelim;
@ -714,7 +746,8 @@ public void setFieldsTerminatedBy(char c) {
/**
* @return the character to print between records when importing them to text.
* @return the character to print between records when importing them to
* text.
*/
public char getOutputRecordDelim() {
return this.outputRecordDelim;
@ -725,7 +758,8 @@ public void setLinesTerminatedBy(char c) {
}
/**
* @return a character which may enclose the contents of fields when imported to text.
* @return a character which may enclose the contents of fields when
* imported to text.
*/
public char getOutputEnclosedBy() {
return this.outputEnclosedBy;
@ -736,7 +770,8 @@ public void setEnclosedBy(char c) {
}
/**
* @return a character which signifies an escape sequence when importing to text.
* @return a character which signifies an escape sequence when importing to
* text.
*/
public char getOutputEscapedBy() {
return this.outputEscapedBy;
@ -747,8 +782,9 @@ public void setEscapedBy(char c) {
}
/**
* @return true if fields imported to text must be enclosed by the EnclosedBy char.
* default is false; set to true if --enclosed-by is used instead of --optionally-enclosed-by.
* @return true if fields imported to text must be enclosed by the
* EnclosedBy char. default is false; set to true if --enclosed-by is used
* instead of --optionally-enclosed-by.
*/
public boolean isOutputEncloseRequired() {
return this.outputMustBeEnclosed;
@ -770,7 +806,7 @@ public void setUseCompression(boolean useCompression) {
}
/**
* @return the name of the destination table when importing to Hive
* @return the name of the destination table when importing to Hive.
*/
public String getHiveTableName() {
if (null != this.hiveTableName) {
@ -829,7 +865,7 @@ public void setConf(Configuration config) {
}
/**
* @return command-line arguments after a '-'
* @return command-line arguments after a '-'.
*/
public String [] getExtraArgs() {
if (extraArgs == null) {
@ -855,3 +891,4 @@ public void setExtraArgs(String [] args) {
}
}
}

2
src/java/org/apache/hadoop/sqoop/cli/SqoopParser.java
View File

@ -51,7 +51,7 @@ public class SqoopParser extends GnuParser {
// this Sqoop class, we cannot see their package-specific methods.
// So we just call it by reflection. As long as we're at it, this
// allows us to also put SqoopParser in its own package.
static java.lang.reflect.Method addValForProcessing;
private static java.lang.reflect.Method addValForProcessing;
static {
try {

2
src/java/org/apache/hadoop/sqoop/cli/ToolOptions.java
View File

@ -114,7 +114,7 @@ public void printHelp() {
}
/**
* Print the help to the console using the specified help formatter
* Print the help to the console using the specified help formatter.
* @param formatter the HelpFormatter to use.
*/
public void printHelp(HelpFormatter formatter) {

10
src/java/org/apache/hadoop/sqoop/hive/HiveImport.java
View File

@ -86,7 +86,7 @@ private String getHiveBinPath() {
/**
* If we used a MapReduce-based upload of the data, remove the _logs dir
* from where we put it, before running Hive LOAD DATA INPATH
* from where we put it, before running Hive LOAD DATA INPATH.
*/
private void removeTempLogs(String tableName) throws IOException {
FileSystem fs = FileSystem.get(configuration);
@ -102,7 +102,8 @@ private void removeTempLogs(String tableName) throws IOException {
if (fs.exists(logsPath)) {
LOG.info("Removing temporary files from import process: " + logsPath);
if (!fs.delete(logsPath, true)) {
LOG.warn("Could not delete temporary files; continuing with import, but it may fail.");
LOG.warn("Could not delete temporary files; "
+ "continuing with import, but it may fail.");
}
}
}
@ -124,7 +125,7 @@ private boolean isGenerateOnly() {
*/
private File getScriptFile(String outputTableName) throws IOException {
if (!isGenerateOnly()) {
return File.createTempFile("hive-script-",".txt",
return File.createTempFile("hive-script-", ".txt",
new File(options.getTempDir()));
} else {
return new File(new File(options.getCodeOutputDir()),
@ -194,7 +195,8 @@ public void importTable(String inputTableName, String outputTableName,
try {
w.close();
} catch (IOException ioe) {
LOG.warn("IOException closing stream to Hive script: " + ioe.toString());
LOG.warn("IOException closing stream to Hive script: "
+ ioe.toString());
}
}
}

5
src/java/org/apache/hadoop/sqoop/hive/HiveTypes.java
View File

@ -26,10 +26,13 @@
/**
* Defines conversion between SQL types and Hive types.
*/
public class HiveTypes {
public final class HiveTypes {
public static final Log LOG = LogFactory.getLog(HiveTypes.class.getName());
private HiveTypes() {
}
/**
* Given JDBC SQL types coming from another database, what is the best
* mapping to a Hive-specific type?

16
src/java/org/apache/hadoop/sqoop/hive/TableDefWriter.java
View File

@ -45,7 +45,8 @@
*/
public class TableDefWriter {
public static final Log LOG = LogFactory.getLog(TableDefWriter.class.getName());
public static final Log LOG = LogFactory.getLog(
TableDefWriter.class.getName());
private SqoopOptions options;
private ConnManager connManager;
@ -139,13 +140,15 @@ public String getCreateTableStmt() throws IOException {
Integer colType = columnTypes.get(col);
String hiveColType = connManager.toHiveType(colType);
if (null == hiveColType) {
throw new IOException("Hive does not support the SQL type for column " + col);
throw new IOException("Hive does not support the SQL type for column "
+ col);
}
sb.append(col + " " + hiveColType);
if (HiveTypes.isHiveTypeImprovised(colType)) {
LOG.warn("Column " + col + " had to be cast to a less precise type in Hive");
LOG.warn(
"Column " + col + " had to be cast to a less precise type in Hive");
}
}
@ -171,7 +174,7 @@ public String getCreateTableStmt() throws IOException {
org.apache.hadoop.hdfs.server.namenode.NameNode.DEFAULT_PORT;
/**
* @return the LOAD DATA statement to import the data in HDFS into hive
* @return the LOAD DATA statement to import the data in HDFS into hive.
*/
public String getLoadDataStmt() throws IOException {
String warehouseDir = options.getWarehouseDir();
@ -207,10 +210,9 @@ public String getLoadDataStmt() throws IOException {
* @param charNum the character to use as a delimiter
* @return a string of the form "\ooo" where ooo is an octal number
* in [000, 177].
* @throws IllegalArgumentException if charNum &gt;> 0177.
* @throws IllegalArgumentException if charNum &gt; 0177.
*/
static String getHiveOctalCharCode(int charNum)
throws IllegalArgumentException {
static String getHiveOctalCharCode(int charNum) {
if (charNum > 0177) {
throw new IllegalArgumentException(
"Character " + charNum + " is an out-of-range delimiter");

1
src/java/org/apache/hadoop/sqoop/io/FixedLengthInputStream.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.sqoop.io;
import java.io.FilterInputStream;
import java.io.InputStream;
import java.io.IOException;

30
src/java/org/apache/hadoop/sqoop/io/LobFile.java
View File

@ -76,7 +76,10 @@
* The LobFile format is specified at:
* http://wiki.github.com/cloudera/sqoop/sip-3
*/
public class LobFile {
public final class LobFile {
private LobFile() {
}
public static final Log LOG = LogFactory.getLog(LobFile.class.getName());
@ -716,7 +719,7 @@ public Iterator<IndexTableEntry> iterator() {
/**
* Class that writes out a LobFile. Instantiate via LobFile.create().
*/
public static abstract class Writer implements Closeable {
public abstract static class Writer implements Closeable {
/**
* If this Writer is writing to a physical LobFile, then this returns
@ -733,6 +736,7 @@ public static abstract class Writer implements Closeable {
@Override
protected synchronized void finalize() throws Throwable {
close();
super.finalize();
}
/**
@ -759,7 +763,7 @@ public abstract java.io.Writer writeClobRecord(long len)
throws IOException;
/**
* Report the current position in the output file
* Report the current position in the output file.
* @return the number of bytes written through this Writer.
*/
public abstract long tell() throws IOException;
@ -795,7 +799,8 @@ private static class V0Writer extends Writer {
// The LobIndex we are constructing.
private LinkedList<IndexSegment> indexSegments;
private int entriesInSegment; // number of entries in the current IndexSegment.
// Number of entries in the current IndexSegment.
private int entriesInSegment;
private IndexTable indexTable;
// Number of entries that can be written to a single IndexSegment.
@ -1078,7 +1083,7 @@ public java.io.Writer writeClobRecord(long len) throws IOException {
/**
* Class that can read a LobFile. Create with LobFile.open().
*/
public static abstract class Reader implements Closeable {
public abstract static class Reader implements Closeable {
/**
* If this Reader is reading from a physical LobFile, then this returns
* the file path it is reading from. Otherwise it returns null.
@ -1087,7 +1092,7 @@ public static abstract class Reader implements Closeable {
public abstract Path getPath();
/**
* Report the current position in the file
* Report the current position in the file.
* @return the current offset from the start of the file in bytes.
*/
public abstract long tell() throws IOException;
@ -1179,6 +1184,7 @@ protected void checkForNull(InputStream in) throws IOException {
@Override
protected synchronized void finalize() throws Throwable {
close();
super.finalize();
}
}
@ -1449,17 +1455,20 @@ private int findRecordStartMark(byte [] buf) {
return -1; // couldn't find it.
}
@Override
/** {@inheritDoc} */
public Path getPath() {
return this.path;
}
@Override
/** {@inheritDoc} */
public long tell() throws IOException {
checkForNull(this.underlyingInput);
return this.underlyingInput.getPos();
}
@Override
/** {@inheritDoc} */
public void seek(long pos) throws IOException {
closeUserStream();
@ -1576,6 +1585,7 @@ private void closeUserStream() throws IOException {
}
}
@Override
/** {@inheritDoc} */
public boolean next() throws IOException {
LOG.debug("Checking for next record");
@ -1646,26 +1656,31 @@ public boolean next() throws IOException {
return true;
}
@Override
/** {@inheritDoc} */
public boolean isRecordAvailable() {
return this.isAligned;
}
@Override
/** {@inheritDoc} */
public long getRecordLen() {
return this.claimedRecordLen;
}
@Override
/** {@inheritDoc} */
public long getRecordId() {
return this.curEntryId;
}
@Override
/** {@inheritDoc} */
public long getRecordOffset() {
return this.curRecordOffset;
}
@Override
/** {@inheritDoc} */
public InputStream readBlobRecord() throws IOException {
if (!isRecordAvailable()) {
@ -1700,6 +1715,7 @@ public InputStream readBlobRecord() throws IOException {
return this.userInputStream;
}
@Override
/** {@inheritDoc} */
public java.io.Reader readClobRecord() throws IOException {
// Get a handle to the binary reader and then wrap it.
@ -1707,6 +1723,7 @@ public java.io.Reader readClobRecord() throws IOException {
return new InputStreamReader(is);
}
@Override
/** {@inheritDoc} */
public void close() throws IOException {
closeUserStream();
@ -1724,6 +1741,7 @@ public void close() throws IOException {
this.isAligned = false;
}
@Override
/** {@inheritDoc} */
public boolean isClosed() {
return this.underlyingInput == null;

4
src/java/org/apache/hadoop/sqoop/io/SplittableBufferedWriter.java
View File

@ -19,10 +19,8 @@
package org.apache.hadoop.sqoop.io;
import java.io.BufferedWriter;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.IOException;
import java.util.Formatter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -48,7 +46,7 @@ public SplittableBufferedWriter(
this.alwaysFlush = false;
}
/** For testing */
/** For testing. */
SplittableBufferedWriter(final SplittingOutputStream splitOutputStream,
final boolean alwaysFlush) {
super(new OutputStreamWriter(splitOutputStream));

8
src/java/org/apache/hadoop/sqoop/lib/BigDecimalSerializer.java
View File

@ -35,7 +35,7 @@
* We serialize in one of two formats:
*
* First, check whether the BigInt can fit in a long:
* boolean b = BigIntegerPart > LONG_MAX || BigIntegerPart < LONG_MIN
* boolean b = BigIntegerPart &gt; LONG_MAX || BigIntegerPart &lt; LONG_MIN
*
* [int: scale][boolean: b == false][long: BigInt-part]
* [int: scale][boolean: b == true][string: BigInt-part.toString()]
@ -46,8 +46,10 @@ public final class BigDecimalSerializer {
private BigDecimalSerializer() { }
static final BigInteger LONG_MAX_AS_BIGINT = BigInteger.valueOf(Long.MAX_VALUE);
static final BigInteger LONG_MIN_AS_BIGINT = BigInteger.valueOf(Long.MIN_VALUE);
static final BigInteger LONG_MAX_AS_BIGINT =
BigInteger.valueOf(Long.MAX_VALUE);
static final BigInteger LONG_MIN_AS_BIGINT =
BigInteger.valueOf(Long.MIN_VALUE);
public static void write(BigDecimal d, DataOutput out) throws IOException {
int scale = d.scale();

23
src/java/org/apache/hadoop/sqoop/lib/BlobRef.java
View File

@ -25,19 +25,9 @@
import java.io.InputStream;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.sqoop.io.LobFile;
import org.apache.hadoop.sqoop.io.LobReaderCache;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -85,7 +75,7 @@ protected InputStream getInternalSource(BytesWritable data) {
}
@Override
protected BytesWritable deepCopyData() {
protected BytesWritable deepCopyData(BytesWritable data) {
return new BytesWritable(Arrays.copyOf(data.getBytes(), data.getLength()));
}
@ -94,15 +84,18 @@ public void readFieldsInternal(DataInput in) throws IOException {
// For internally-stored BLOBs, the data is a BytesWritable
// containing the actual data.
if (null == this.data) {
this.data = new BytesWritable();
BytesWritable data = getDataObj();
if (null == data) {
data = new BytesWritable();
}
this.data.readFields(in);
data.readFields(in);
setDataObj(data);
}
@Override
public void writeInternal(DataOutput out) throws IOException {
data.write(out);
getDataObj().write(out);
}
/**

15
src/java/org/apache/hadoop/sqoop/lib/ClobRef.java
View File

@ -21,20 +21,11 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.sqoop.io.LobFile;
/**
@ -73,7 +64,7 @@ protected Reader getInternalSource(String data) {
}
@Override
protected String deepCopyData() {
protected String deepCopyData(String data) {
return data;
}
@ -85,12 +76,12 @@ protected String getInternalData(String data) {
@Override
public void readFieldsInternal(DataInput in) throws IOException {
// For internally-stored clobs, the data is written as UTF8 Text.
this.data = Text.readString(in);
setDataObj(Text.readString(in));
}
@Override
public void writeInternal(DataOutput out) throws IOException {
Text.writeString(out, data);
Text.writeString(out, getDataObj());
}
/**

41
src/java/org/apache/hadoop/sqoop/lib/FieldFormatter.java
View File

@ -28,31 +28,35 @@ private FieldFormatter() { }
/**
* Takes an input string representing the value of a field, encloses it in
* enclosing chars, and escapes any occurrences of such characters in the middle.
* The escape character itself is also escaped if it appears in the text of the
* field.
* enclosing chars, and escapes any occurrences of such characters in the
* middle. The escape character itself is also escaped if it appears in the
* text of the field.
*
* The field is enclosed only if:
* enclose != '\000', and:
* encloseRequired is true, or
* one of the characters in the mustEscapeFor list is present in the string.
* one of the characters in the mustEscapeFor list is present
* in the string.
*
* Escaping is not performed if the escape char is '\000'.
*
* @param str - The user's string to escape and enclose
* @param escape - What string to use as the escape sequence. If "" or null, then don't escape.
* @param enclose - The string to use to enclose str e.g. "quoted". If "" or null, then don't
* enclose.
* @param mustEncloseFor - A list of characters; if one is present in 'str', then str must be
* enclosed
* @param encloseRequired - If true, then always enclose, regardless of mustEscapeFor
* @return the escaped, enclosed version of 'str'
* @param escape - What string to use as the escape sequence. If "" or null,
* then don't escape.
* @param enclose - The string to use to enclose str e.g. "quoted". If "" or
* null, then don't enclose.
* @param mustEncloseFor - A list of characters; if one is present in 'str',
* then str must be enclosed.
* @param encloseRequired - If true, then always enclose, regardless of
* mustEscapeFor.
* @return the escaped, enclosed version of 'str'.
*/
public static final String escapeAndEnclose(String str, String escape, String enclose,
char [] mustEncloseFor, boolean encloseRequired) {
public static String escapeAndEnclose(String str, String escape,
String enclose, char [] mustEncloseFor, boolean encloseRequired) {
// true if we can use an escape character.
boolean escapingLegal = (null != escape && escape.length() > 0 && !escape.equals("\000"));
boolean escapingLegal = (null != escape
&& escape.length() > 0 && !escape.equals("\000"));
String withEscapes;
if (null == str) {
@ -60,7 +64,7 @@ public static final String escapeAndEnclose(String str, String escape, String en
}
if (escapingLegal) {
// escaping is legal. Escape any instances of the escape char itself
// escaping is legal. Escape any instances of the escape char itself.
withEscapes = str.replace(escape, escape + escape);
} else {
// no need to double-escape
@ -68,12 +72,13 @@ public static final String escapeAndEnclose(String str, String escape, String en
}
if (null == enclose || enclose.length() == 0 || enclose.equals("\000")) {
// The enclose-with character was left unset, so we can't enclose items. We're done.
// The enclose-with character was left unset, so we can't enclose items.
// We're done.
return withEscapes;
}
// if we have an enclosing character, and escaping is legal, then the encloser must
// always be escaped.
// if we have an enclosing character, and escaping is legal, then the
// encloser must always be escaped.
if (escapingLegal) {
withEscapes = withEscapes.replace(enclose, escape + enclose);
}

61
src/java/org/apache/hadoop/sqoop/lib/JdbcWritableBridge.java
View File

@ -20,8 +20,6 @@
import org.apache.hadoop.io.BytesWritable;
import java.math.BigDecimal;
import java.sql.Blob;
import java.sql.Clob;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@ -37,14 +35,16 @@
*/
public final class JdbcWritableBridge {
// Currently, cap BLOB/CLOB objects at 16 MB until we can use external storage.
public final static long MAX_BLOB_LENGTH = 16 * 1024 * 1024;
public final static long MAX_CLOB_LENGTH = 16 * 1024 * 1024;
// Currently, cap BLOB/CLOB objects at 16 MB until we can use external
// storage.
public static final long MAX_BLOB_LENGTH = 16 * 1024 * 1024;
public static final long MAX_CLOB_LENGTH = 16 * 1024 * 1024;
private JdbcWritableBridge() {
}
public static Integer readInteger(int colNum, ResultSet r) throws SQLException {
public static Integer readInteger(int colNum, ResultSet r)
throws SQLException {
int val;
val = r.getInt(colNum);
if (r.wasNull()) {
@ -88,7 +88,8 @@ public static Double readDouble(int colNum, ResultSet r) throws SQLException {
}
}
public static Boolean readBoolean(int colNum, ResultSet r) throws SQLException {
public static Boolean readBoolean(int colNum, ResultSet r)
throws SQLException {
boolean val;
val = r.getBoolean(colNum);
if (r.wasNull()) {
@ -102,7 +103,8 @@ public static Time readTime(int colNum, ResultSet r) throws SQLException {
return r.getTime(colNum);
}
public static Timestamp readTimestamp(int colNum, ResultSet r) throws SQLException {
public static Timestamp readTimestamp(int colNum, ResultSet r)
throws SQLException {
return r.getTimestamp(colNum);
}
@ -116,7 +118,8 @@ public static BytesWritable readBytesWritable(int colNum, ResultSet r)
return new BytesWritable(bytes);
}
public static BigDecimal readBigDecimal(int colNum, ResultSet r) throws SQLException {
public static BigDecimal readBigDecimal(int colNum, ResultSet r)
throws SQLException {
return r.getBigDecimal(colNum);
}
@ -132,8 +135,8 @@ public static ClobRef readClobRef(int colNum, ResultSet r)
return null;
}
public static void writeInteger(Integer val, int paramIdx, int sqlType, PreparedStatement s)
throws SQLException {
public static void writeInteger(Integer val, int paramIdx, int sqlType,
PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {
@ -141,8 +144,8 @@ public static void writeInteger(Integer val, int paramIdx, int sqlType, Prepared
}
}
public static void writeLong(Long val, int paramIdx, int sqlType, PreparedStatement s)
throws SQLException {
public static void writeLong(Long val, int paramIdx, int sqlType,
PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {
@ -150,8 +153,8 @@ public static void writeLong(Long val, int paramIdx, int sqlType, PreparedStatem
}
}
public static void writeDouble(Double val, int paramIdx, int sqlType, PreparedStatement s)
throws SQLException {
public static void writeDouble(Double val, int paramIdx, int sqlType,
PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {
@ -159,8 +162,8 @@ public static void writeDouble(Double val, int paramIdx, int sqlType, PreparedSt
}
}
public static void writeBoolean(Boolean val, int paramIdx, int sqlType, PreparedStatement s)
throws SQLException {
public static void writeBoolean(Boolean val, int paramIdx, int sqlType,
PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {
@ -168,8 +171,8 @@ public static void writeBoolean(Boolean val, int paramIdx, int sqlType, Prepared
}
}
public static void writeFloat(Float val, int paramIdx, int sqlType, PreparedStatement s)
throws SQLException {
public static void writeFloat(Float val, int paramIdx, int sqlType,
PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {
@ -177,8 +180,8 @@ public static void writeFloat(Float val, int paramIdx, int sqlType, PreparedStat
}
}
public static void writeString(String val, int paramIdx, int sqlType, PreparedStatement s)
throws SQLException {
public static void writeString(String val, int paramIdx, int sqlType,
PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {
@ -186,8 +189,8 @@ public static void writeString(String val, int paramIdx, int sqlType, PreparedSt
}
}
public static void writeTimestamp(Timestamp val, int paramIdx, int sqlType, PreparedStatement s)
throws SQLException {
public static void writeTimestamp(Timestamp val, int paramIdx, int sqlType,
PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {
@ -195,8 +198,8 @@ public static void writeTimestamp(Timestamp val, int paramIdx, int sqlType, Prep
}
}
public static void writeTime(Time val, int paramIdx, int sqlType, PreparedStatement s)
throws SQLException {
public static void writeTime(Time val, int paramIdx, int sqlType,
PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {
@ -204,8 +207,8 @@ public static void writeTime(Time val, int paramIdx, int sqlType, PreparedStatem
}
}
public static void writeDate(Date val, int paramIdx, int sqlType, PreparedStatement s)
throws SQLException {
public static void writeDate(Date val, int paramIdx, int sqlType,
PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {
@ -228,8 +231,8 @@ public static void writeBytesWritable(BytesWritable val, int paramIdx,
}
public static void writeBigDecimal(BigDecimal val, int paramIdx, int sqlType, PreparedStatement s)
throws SQLException {
public static void writeBigDecimal(BigDecimal val, int paramIdx,
int sqlType, PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {

16
src/java/org/apache/hadoop/sqoop/lib/LargeObjectLoader.java
View File

@ -18,30 +18,21 @@
package org.apache.hadoop.sqoop.lib;
import java.io.BufferedOutputStream;
import java.io.BufferedWriter;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.math.BigDecimal;
import java.sql.Blob;
import java.sql.Clob;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.sqoop.io.LobFile;
/**
@ -57,9 +48,9 @@
public class LargeObjectLoader implements Closeable {
// Spill to external storage for BLOB/CLOB objects > 16 MB.
public final static long DEFAULT_MAX_LOB_LENGTH = 16 * 1024 * 1024;
public static final long DEFAULT_MAX_LOB_LENGTH = 16 * 1024 * 1024;
public final static String MAX_INLINE_LOB_LEN_KEY =
public static final String MAX_INLINE_LOB_LEN_KEY =
"sqoop.inline.lob.length.max";
private Configuration conf;
@ -75,7 +66,7 @@ public class LargeObjectLoader implements Closeable {
private long nextLobFileId = 0;
/**
* Create a new LargeObjectLoader
* Create a new LargeObjectLoader.
* @param conf the Configuration to use
* @param workPath the HDFS working directory for this task.
*/
@ -91,6 +82,7 @@ public LargeObjectLoader(Configuration conf, Path workPath)
@Override
protected synchronized void finalize() throws Throwable {
close();
super.finalize();
}
@Override

67
src/java/org/apache/hadoop/sqoop/lib/LobRef.java
View File

@ -22,12 +22,10 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.InputStream;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
@ -57,7 +55,7 @@ protected LobRef() {
this.offset = 0;
this.length = 0;
this.data = null;
this.realData = null;
}
protected LobRef(CONTAINERTYPE container) {
@ -65,7 +63,7 @@ protected LobRef(CONTAINERTYPE container) {
this.offset = 0;
this.length = 0;
this.data = container;
this.realData = container;
}
protected LobRef(String file, long offset, long length) {
@ -73,11 +71,21 @@ protected LobRef(String file, long offset, long length) {
this.offset = offset;
this.length = length;
this.data = null;
this.realData = null;
}
// If the data is 'small', it's held directly, here.
protected CONTAINERTYPE data;
private CONTAINERTYPE realData;
/** Internal API to retrieve the data object. */
protected CONTAINERTYPE getDataObj() {
return realData;
}
/** Internal API to set the data object. */
protected void setDataObj(CONTAINERTYPE data) {
this.realData = data;
}
// If there data is too large to materialize fully, it's written into a file
// whose path (relative to the rest of the dataset) is recorded here. This
@ -90,7 +98,7 @@ protected LobRef(String file, long offset, long length) {
private long length;
// If we've opened a LobFile object, track our reference to it here.
protected LobFile.Reader reader;
private LobFile.Reader lobReader;
@Override
@SuppressWarnings("unchecked")
@ -102,9 +110,9 @@ public Object clone() throws CloneNotSupportedException {
LobRef<DATATYPE, CONTAINERTYPE, ACCESSORTYPE> r =
(LobRef<DATATYPE, CONTAINERTYPE, ACCESSORTYPE>) super.clone();
r.reader = null; // Reference to opened reader is not duplicated.
if (null != data) {
r.data = deepCopyData();
r.lobReader = null; // Reference to opened reader is not duplicated.
if (null != realData) {
r.realData = deepCopyData(realData);
}
return r;
@ -113,12 +121,13 @@ public Object clone() throws CloneNotSupportedException {
@Override
protected synchronized void finalize() throws Throwable {
close();
super.finalize();
}
public void close() throws IOException {
// Discard any open LobReader.
if (null != this.reader) {
LobReaderCache.getCache().recycle(this.reader);
if (null != this.lobReader) {
LobReaderCache.getCache().recycle(this.lobReader);
}
}
@ -142,7 +151,7 @@ public boolean isExternal() {
* @throws IOException if it could not read the LOB from external storage.
*/
public ACCESSORTYPE getDataStream(Mapper.Context mapContext)
throws IllegalArgumentException, IOException {
throws IOException {
InputSplit split = mapContext.getInputSplit();
if (split instanceof FileSplit) {
Path basePath = ((FileSplit) split).getPath().getParent();
@ -171,35 +180,35 @@ public ACCESSORTYPE getDataStream(Configuration conf, Path basePath)
Path pathToRead = LobReaderCache.qualify(
new Path(basePath, fileName), conf);
LOG.debug("Retreving data stream from external path: " + pathToRead);
if (reader != null) {
if (lobReader != null) {
// We already have a reader open to a LobFile. Is it the correct file?
if (!pathToRead.equals(reader.getPath())) {
// No. Close this reader and get the correct one.
if (!pathToRead.equals(lobReader.getPath())) {
// No. Close this.lobReader and get the correct one.
LOG.debug("Releasing previous external reader for "
+ reader.getPath());
LobReaderCache.getCache().recycle(reader);
reader = LobReaderCache.getCache().get(pathToRead, conf);
+ lobReader.getPath());
LobReaderCache.getCache().recycle(lobReader);
lobReader = LobReaderCache.getCache().get(pathToRead, conf);
}
} else {
reader = LobReaderCache.getCache().get(pathToRead, conf);
lobReader = LobReaderCache.getCache().get(pathToRead, conf);
}
// We now have a LobFile.Reader associated with the correct file. Get to
// the correct offset and return an InputStream/Reader to the user.
if (reader.tell() != offset) {
if (lobReader.tell() != offset) {
LOG.debug("Seeking to record start offset " + offset);
reader.seek(offset);
lobReader.seek(offset);
}
if (!reader.next()) {
if (!lobReader.next()) {
throw new IOException("Could not locate record at " + pathToRead
+ ":" + offset);
}
return getExternalSource(reader);
return getExternalSource(lobReader);
} else {
// This data is already materialized in memory; wrap it and return.
return getInternalSource(data);
return getInternalSource(realData);
}
}
@ -223,7 +232,7 @@ protected abstract ACCESSORTYPE getExternalSource(LobFile.Reader reader)
/**
* Make a copy of the materialized data.
*/
protected abstract CONTAINERTYPE deepCopyData();
protected abstract CONTAINERTYPE deepCopyData(CONTAINERTYPE data);
public DATATYPE getData() {
if (isExternal()) {
@ -231,7 +240,7 @@ public DATATYPE getData() {
"External LOBs must be read via getDataStream()");
}
return getInternalData(data);
return getInternalData(realData);
}
@Override
@ -240,7 +249,7 @@ public String toString() {
return "externalLob(lf," + fileName + "," + Long.toString(offset)
+ "," + Long.toString(length) + ")";
} else {
return data.toString();
return realData.toString();
}
}
@ -260,7 +269,7 @@ public void readFields(DataInput in) throws IOException {
boolean isExternal = in.readBoolean();
if (isExternal) {
this.data = null;
this.realData = null;
String storageType = Text.readString(in);
if (!storageType.equals("lf")) {

6
src/java/org/apache/hadoop/sqoop/lib/LobSerializer.java
View File

@ -29,11 +29,13 @@ public final class LobSerializer {
private LobSerializer() { }
public static void writeClob(ClobRef clob, DataOutput out) throws IOException {
public static void writeClob(ClobRef clob, DataOutput out)
throws IOException {
clob.write(out);
}
public static void writeBlob(BlobRef blob, DataOutput out) throws IOException {
public static void writeBlob(BlobRef blob, DataOutput out)
throws IOException {
blob.write(out);
}

22
src/java/org/apache/hadoop/sqoop/lib/RecordParser.java
View File

@ -65,6 +65,9 @@ private enum ParseState {
UNENCLOSED_ESCAPE
}
/**
* An error thrown when parsing fails.
*/
public static class ParseError extends Exception {
public ParseError() {
super("ParseError");
@ -246,7 +249,8 @@ record sep halts processing.
sb.append(curChar);
if (this.enclosingRequired) {
throw new ParseError("Opening field-encloser expected at position " + pos);
throw new ParseError(
"Opening field-encloser expected at position " + pos);
}
}
@ -285,15 +289,15 @@ record sep halts processing.
break;
case ENCLOSED_ESCAPE:
// Treat this character literally, whatever it is, and return to enclosed
// field processing.
// Treat this character literally, whatever it is, and return to
// enclosed field processing.
sb.append(curChar);
state = ParseState.ENCLOSED_FIELD;
break;
case ENCLOSED_EXPECT_DELIMITER:
// We were in an enclosed field, but got the final encloser. Now we expect
// either an end-of-field or an end-of-record.
// We were in an enclosed field, but got the final encloser. Now we
// expect either an end-of-field or an end-of-record.
if (this.fieldDelim == curChar) {
// end of one field is the beginning of the next.
state = ParseState.FIELD_START;
@ -308,8 +312,8 @@ record sep halts processing.
break;
case UNENCLOSED_ESCAPE:
// Treat this character literally, whatever it is, and return to non-enclosed
// field processing.
// Treat this character literally, whatever it is, and return to
// non-enclosed field processing.
sb.append(curChar);
state = ParseState.UNENCLOSED_FIELD;
break;
@ -342,8 +346,8 @@ public boolean isEnclosingRequired() {
@Override
public String toString() {
return "RecordParser[" + fieldDelim + ',' + recordDelim + ',' + enclosingChar + ','
+ escapeChar + ',' + enclosingRequired + "]";
return "RecordParser[" + fieldDelim + ',' + recordDelim + ','
+ enclosingChar + ',' + escapeChar + ',' + enclosingRequired + "]";
}
@Override

18
src/java/org/apache/hadoop/sqoop/lib/SqoopRecord.java
View File

@ -32,21 +32,21 @@
* Interface implemented by the classes generated by sqoop's orm.ClassWriter.
*/
public interface SqoopRecord extends Cloneable, DBWritable, Writable {
public void parse(CharSequence s) throws RecordParser.ParseError;
public void parse(Text s) throws RecordParser.ParseError;
public void parse(byte [] s) throws RecordParser.ParseError;
public void parse(char [] s) throws RecordParser.ParseError;
public void parse(ByteBuffer s) throws RecordParser.ParseError;
public void parse(CharBuffer s) throws RecordParser.ParseError;
public void loadLargeObjects(LargeObjectLoader objLoader)
void parse(CharSequence s) throws RecordParser.ParseError;
void parse(Text s) throws RecordParser.ParseError;
void parse(byte [] s) throws RecordParser.ParseError;
void parse(char [] s) throws RecordParser.ParseError;
void parse(ByteBuffer s) throws RecordParser.ParseError;
void parse(CharBuffer s) throws RecordParser.ParseError;
void loadLargeObjects(LargeObjectLoader objLoader)
throws SQLException, IOException, InterruptedException;
public Object clone() throws CloneNotSupportedException;
Object clone() throws CloneNotSupportedException;
/**
* Inserts the data in this object into the PreparedStatement, starting
* at parameter 'offset'.
* @return the number of fields written to the statement.
*/
public int write(PreparedStatement stmt, int offset) throws SQLException;
int write(PreparedStatement stmt, int offset) throws SQLException;
}

22
src/java/org/apache/hadoop/sqoop/manager/ConnManager.java
View File

@ -37,12 +37,12 @@
public abstract class ConnManager {
/**
* Return a list of all databases on a server
* Return a list of all databases on a server.
*/
public abstract String [] listDatabases();
/**
* Return a list of all tables in a database
* Return a list of all tables in a database.
*/
public abstract String [] listTables();
@ -57,14 +57,14 @@ public abstract class ConnManager {
public abstract String getPrimaryKey(String tableName);
/**
* Return java type for SQL type
* Return java type for SQL type.
* @param sqlType sql type
* @return java type
*/
public abstract String toJavaType(int sqlType);
/**
* Return hive type for SQL type
* Return hive type for SQL type.
* @param sqlType sql type
* @return hive type
*/
@ -86,25 +86,27 @@ public abstract class ConnManager {
* returned ResultSet object, and for calling release() after that to free
* internal state.
*/
public abstract ResultSet readTable(String tableName, String [] columns) throws SQLException;
public abstract ResultSet readTable(String tableName, String [] columns)
throws SQLException;
/**
* @return the actual database connection
* @return the actual database connection.
*/
public abstract Connection getConnection() throws SQLException;
/**
* @return a string identifying the driver class to load for this JDBC connection type.
* @return a string identifying the driver class to load for this
* JDBC connection type.
*/
public abstract String getDriverClass();
/**
* Execute a SQL statement 's' and print its results to stdout
* Execute a SQL statement 's' and print its results to stdout.
*/
public abstract void execAndPrint(String s);
/**
* Perform an import of a table from the database into HDFS
* Perform an import of a table from the database into HDFS.
*/
public abstract void importTable(ImportJobContext context)
throws IOException, ImportException;
@ -139,7 +141,7 @@ public String escapeTableName(String tableName) {
public abstract void close() throws SQLException;
/**
* Export data stored in HDFS into a table in a database
* Export data stored in HDFS into a table in a database.
*/
public void exportTable(ExportJobContext context)
throws IOException, ExportException {

3
src/java/org/apache/hadoop/sqoop/manager/DefaultManagerFactory.java
View File

@ -29,7 +29,8 @@
*/
public final class DefaultManagerFactory extends ManagerFactory {
public static final Log LOG = LogFactory.getLog(DefaultManagerFactory.class.getName());
public static final Log LOG = LogFactory.getLog(
DefaultManagerFactory.class.getName());
public ConnManager accept(SqoopOptions options) {
String manualDriver = options.getDriverClassName();

3
src/java/org/apache/hadoop/sqoop/manager/DirectMySQLManager.java
View File

@ -46,7 +46,8 @@
*/
public class DirectMySQLManager extends MySQLManager {
public static final Log LOG = LogFactory.getLog(DirectMySQLManager.class.getName());
public static final Log LOG = LogFactory.getLog(
DirectMySQLManager.class.getName());
public DirectMySQLManager(final SqoopOptions options) {
super(options, false);

67
src/java/org/apache/hadoop/sqoop/manager/DirectPostgresqlManager.java
View File

@ -51,10 +51,12 @@
* commands.
*/
public class DirectPostgresqlManager extends PostgresqlManager {
public static final Log LOG = LogFactory.getLog(DirectPostgresqlManager.class.getName());
public static final Log LOG = LogFactory.getLog(
DirectPostgresqlManager.class.getName());
public DirectPostgresqlManager(final SqoopOptions opts) {
// Inform superclass that we're overriding import method via alt. constructor.
// Inform superclass that we're overriding import method via alt.
// constructor.
super(opts, true);
}
@ -68,8 +70,8 @@ static class PostgresqlAsyncSink extends ErrorableAsyncSink {
private final PerfCounters counters;
private final SqoopOptions options;
PostgresqlAsyncSink(final SplittableBufferedWriter w, final SqoopOptions opts,
final PerfCounters ctrs) {
PostgresqlAsyncSink(final SplittableBufferedWriter w,
final SqoopOptions opts, final PerfCounters ctrs) {
this.writer = w;
this.options = opts;
this.counters = ctrs;
@ -81,14 +83,16 @@ public void processStream(InputStream is) {
}
private static class PostgresqlStreamThread extends ErrorableThread {
public static final Log LOG = LogFactory.getLog(PostgresqlStreamThread.class.getName());
public static final Log LOG = LogFactory.getLog(
PostgresqlStreamThread.class.getName());
private final SplittableBufferedWriter writer;
private final InputStream stream;
private final SqoopOptions options;
private final PerfCounters counters;
PostgresqlStreamThread(final InputStream is, final SplittableBufferedWriter w,
PostgresqlStreamThread(final InputStream is,
final SplittableBufferedWriter w,
final SqoopOptions opts, final PerfCounters ctrs) {
this.stream = is;
this.writer = w;
@ -143,7 +147,8 @@ public void run() {
}
/**
Takes a list of columns and turns them into a string like "col1, col2, col3..."
* Takes a list of columns and turns them into a string like
* "col1, col2, col3...".
*/
private String getColumnListStr(String [] cols) {
if (null == cols) {
@ -164,20 +169,20 @@ private String getColumnListStr(String [] cols) {
}
/**
* @return the Postgresql-specific SQL command to copy the table ("COPY .... TO STDOUT")
* @return the Postgresql-specific SQL command to copy the
* table ("COPY .... TO STDOUT").
*/
private String getCopyCommand(String tableName) {
/*
Format of this command is:
COPY table(col, col....) TO STDOUT
or COPY ( query ) TO STDOUT
WITH DELIMITER 'fieldsep'
CSV
QUOTE 'quotechar'
ESCAPE 'escapechar'
FORCE QUOTE col, col, col....
*/
// Format of this command is:
//
// COPY table(col, col....) TO STDOUT
// or COPY ( query ) TO STDOUT
// WITH DELIMITER 'fieldsep'
// CSV
// QUOTE 'quotechar'
// ESCAPE 'escapechar'
// FORCE QUOTE col, col, col....
StringBuilder sb = new StringBuilder();
String [] cols = getColumnNames(tableName);
@ -238,12 +243,12 @@ or COPY ( query ) TO STDOUT
return copyCmd;
}
/** Write the COPY command to a temp file
/** Write the COPY command to a temp file.
* @return the filename we wrote to.
*/
private String writeCopyCommand(String command) throws IOException {
String tmpDir = options.getTempDir();
File tempFile = File.createTempFile("tmp-",".sql", new File(tmpDir));
File tempFile = File.createTempFile("tmp-", ".sql", new File(tmpDir));
BufferedWriter w = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(tempFile)));
w.write(command);
@ -258,7 +263,7 @@ private String writeCopyCommand(String command) throws IOException {
private String writePasswordFile(String password) throws IOException {
String tmpDir = options.getTempDir();
File tempFile = File.createTempFile("pgpass",".pgpass", new File(tmpDir));
File tempFile = File.createTempFile("pgpass", ".pgpass", new File(tmpDir));
LOG.debug("Writing password to tempfile: " + tempFile);
// Make sure it's only readable by the current user.
@ -300,9 +305,9 @@ public void importTable(ImportJobContext context)
PerfCounters counters = new PerfCounters();
try {
// Get the COPY TABLE command to issue, write this to a file, and pass it
// in to psql with -f filename.
// Then make sure we delete this file in our finally block.
// Get the COPY TABLE command to issue, write this to a file, and pass
// it in to psql with -f filename. Then make sure we delete this file
// in our finally block.
String copyCmd = getCopyCommand(tableName);
commandFilename = writeCopyCommand(copyCmd);
@ -312,10 +317,10 @@ public void importTable(ImportJobContext context)
// Environment to pass to psql.
List<String> envp = Executor.getCurEnvpStrings();
// We need to parse the connect string URI to determine the database name
// and the host and port. If the host is localhost and the port is not specified,
// we don't want to pass this to psql, because we want to force the use of a
// UNIX domain socket, not a TCP/IP socket.
// We need to parse the connect string URI to determine the database
// name and the host and port. If the host is localhost and the port is
// not specified, we don't want to pass this to psql, because we want to
// force the use of a UNIX domain socket, not a TCP/IP socket.
String connectString = options.getConnectString();
String databaseName = JdbcUrl.getDatabaseName(connectString);
String hostname = JdbcUrl.getHostName(connectString);
@ -325,7 +330,8 @@ public void importTable(ImportJobContext context)
throw new ImportException("Could not determine database name");
}
LOG.info("Performing import of table " + tableName + " from database " + databaseName);
LOG.info("Performing import of table " + tableName + " from database "
+ databaseName);
args.add(PSQL_CMD); // requires that this is on the path.
args.add("--tuples-only");
args.add("--quiet");
@ -401,7 +407,8 @@ public void importTable(ImportJobContext context)
// Remove any password file we wrote
if (null != passwordFilename) {
if (!new File(passwordFilename).delete()) {
LOG.error("Could not remove postgresql password file " + passwordFilename);
LOG.error("Could not remove postgresql password file "
+ passwordFilename);
LOG.error("You should remove this file to protect your credentials.");
}
}

3
src/java/org/apache/hadoop/sqoop/manager/ExportJobContext.java
View File

@ -30,7 +30,8 @@ public class ExportJobContext {
private String jarFile;
private SqoopOptions options;
public ExportJobContext(final String table, final String jar, final SqoopOptions opts) {
public ExportJobContext(final String table, final String jar,
final SqoopOptions opts) {
this.tableName = table;
this.jarFile = jar;
this.options = opts;

3
src/java/org/apache/hadoop/sqoop/manager/GenericJdbcManager.java
View File

@ -33,7 +33,8 @@
*/
public class GenericJdbcManager extends SqlManager {
public static final Log LOG = LogFactory.getLog(GenericJdbcManager.class.getName());
public static final Log LOG = LogFactory.getLog(
GenericJdbcManager.class.getName());
private String jdbcDriverClass;
private Connection connection;

10
src/java/org/apache/hadoop/sqoop/manager/HsqldbManager.java
View File

@ -29,13 +29,14 @@
*/
public class HsqldbManager extends GenericJdbcManager {
public static final Log LOG = LogFactory.getLog(HsqldbManager.class.getName());
public static final Log LOG = LogFactory.getLog(
HsqldbManager.class.getName());
// driver class to ensure is loaded when making db connection.
private static final String DRIVER_CLASS = "org.hsqldb.jdbcDriver";
// HsqlDb doesn't have a notion of multiple "databases"; the user's database is always called
// "PUBLIC";
// HsqlDb doesn't have a notion of multiple "databases"; the user's database
// is always called "PUBLIC".
private static final String HSQL_SCHEMA_NAME = "PUBLIC";
public HsqldbManager(final SqoopOptions opts) {
@ -43,7 +44,8 @@ public HsqldbManager(final SqoopOptions opts) {
}
/**
* Note: HSqldb only supports a single schema named "PUBLIC"
* Return list of databases hosted by the server.
* HSQLDB only supports a single schema named "PUBLIC".
*/
@Override
public String[] listDatabases() {

3
src/java/org/apache/hadoop/sqoop/manager/ImportJobContext.java
View File

@ -33,7 +33,8 @@ public class ImportJobContext {
private SqoopOptions options;
private Class<? extends InputFormat> inputFormatClass;
public ImportJobContext(final String table, final String jar, final SqoopOptions opts) {
public ImportJobContext(final String table, final String jar,
final SqoopOptions opts) {
this.tableName = table;
this.jarFile = jar;
this.options = opts;

4
src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java
View File

@ -37,7 +37,7 @@
import org.apache.hadoop.sqoop.util.ImportException;
/**
* Manages connections to MySQL databases
* Manages connections to MySQL databases.
*/
public class MySQLManager extends GenericJdbcManager {
@ -103,7 +103,7 @@ public String[] listDatabases() {
@Override
public void importTable(ImportJobContext context)
throws IOException, ImportException {
throws IOException, ImportException {
// Check that we're not doing a MapReduce from localhost. If we are, point
// out that we could use mysqldump.

12
src/java/org/apache/hadoop/sqoop/manager/MySQLUtils.java
View File

@ -18,28 +18,20 @@
package org.apache.hadoop.sqoop.manager;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.CharBuffer;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.sqoop.shims.HadoopShim;
import org.apache.hadoop.sqoop.util.DirectImportUtils;
/**
* Helper methods and constants for MySQL imports/exports
* Helper methods and constants for MySQL imports/exports.
*/
public final class MySQLUtils {
@ -100,7 +92,7 @@ public static String writePasswordFile(Configuration conf)
// Create the temp file to hold the user's password.
String tmpDir = conf.get(
HadoopShim.get().getJobLocalDirProperty(), "/tmp/");
File tempFile = File.createTempFile("mysql-cnf",".cnf", new File(tmpDir));
File tempFile = File.createTempFile("mysql-cnf", ".cnf", new File(tmpDir));
// Make the password file only private readable.
DirectImportUtils.setFilePermissions(tempFile, "0600");

48
src/java/org/apache/hadoop/sqoop/manager/OracleManager.java
View File

@ -47,7 +47,8 @@
*/
public class OracleManager extends GenericJdbcManager {
public static final Log LOG = LogFactory.getLog(OracleManager.class.getName());
public static final Log LOG = LogFactory.getLog(
OracleManager.class.getName());
// driver class to ensure is loaded when making db connection.
private static final String DRIVER_CLASS = "oracle.jdbc.OracleDriver";
@ -65,8 +66,8 @@ private static class ConnCache {
public static final Log LOG = LogFactory.getLog(ConnCache.class.getName());
private static class CacheKey {
public final String connectString;
public final String username;
private final String connectString;
private final String username;
public CacheKey(String connect, String user) {
this.connectString = connect;
@ -212,7 +213,8 @@ protected Connection makeConnection() throws SQLException {
try {
Class.forName(driverClass);
} catch (ClassNotFoundException cnfe) {
throw new RuntimeException("Could not load db driver class: " + driverClass);
throw new RuntimeException("Could not load db driver class: "
+ driverClass);
}
String username = options.getUsername();
@ -242,34 +244,37 @@ protected Connection makeConnection() throws SQLException {
}
/**
* Set session time zone
* Set session time zone.
* @param conn Connection object
* @throws SQLException instance
*/
private void setSessionTimeZone(Connection conn) throws SQLException {
// need to use reflection to call the method setSessionTimeZone on the OracleConnection class
// because oracle specific java libraries are not accessible in this context
// Need to use reflection to call the method setSessionTimeZone on the
// OracleConnection class because oracle specific java libraries are not
// accessible in this context.
Method method;
try {
method = conn.getClass().getMethod(
"setSessionTimeZone", new Class [] {String.class});
} catch (Exception ex) {
LOG.error("Could not find method setSessionTimeZone in " + conn.getClass().getName(), ex);
LOG.error("Could not find method setSessionTimeZone in "
+ conn.getClass().getName(), ex);
// rethrow SQLException
throw new SQLException(ex);
}
// Need to set the time zone in order for Java
// to correctly access the column "TIMESTAMP WITH LOCAL TIME ZONE".
// The user may have set this in the configuration as 'oracle.sessionTimeZone'.
String clientTimeZoneStr = options.getConf().get(ORACLE_TIMEZONE_KEY, "GMT");
// Need to set the time zone in order for Java to correctly access the
// column "TIMESTAMP WITH LOCAL TIME ZONE". The user may have set this in
// the configuration as 'oracle.sessionTimeZone'.
String clientTimeZoneStr = options.getConf().get(ORACLE_TIMEZONE_KEY,
"GMT");
try {
method.setAccessible(true);
method.invoke(conn, clientTimeZoneStr);
LOG.info("Time zone has been set to " + clientTimeZoneStr);
} catch (Exception ex) {
LOG.warn("Time zone " + clientTimeZoneStr +
" could not be set on Oracle database.");
LOG.warn("Time zone " + clientTimeZoneStr
+ " could not be set on Oracle database.");
LOG.info("Setting default time zone: GMT");
try {
// Per the documentation at:
@ -310,7 +315,8 @@ public void exportTable(ExportJobContext context)
}
@Override
public ResultSet readTable(String tableName, String[] columns) throws SQLException {
public ResultSet readTable(String tableName, String[] columns)
throws SQLException {
if (columns == null) {
columns = getColumnNames(tableName);
}
@ -408,14 +414,14 @@ private String dbToHiveType(int sqlType) {
}
/**
* Get database type
* Get database type.
* @param clazz oracle class representing sql types
* @param fieldName field name
* @return value of database type constant
*/
private int getDatabaseType(Class clazz, String fieldName) {
// need to use reflection to extract constant values
// because the database specific java libraries are not accessible in this context
// Need to use reflection to extract constant values because the database
// specific java libraries are not accessible in this context.
int value = -1;
try {
java.lang.reflect.Field field = clazz.getDeclaredField(fieldName);
@ -429,13 +435,13 @@ private int getDatabaseType(Class clazz, String fieldName) {
}
/**
* Load class by name
* Load class by name.
* @param className class name
* @return class instance
*/
private Class getTypeClass(String className) {
// need to use reflection to load class
// because the database specific java libraries are not accessible in this context
// Need to use reflection to load class because the database specific java
// libraries are not accessible in this context.
Class typeClass = null;
try {
typeClass = Class.forName(className);

8
src/java/org/apache/hadoop/sqoop/manager/PostgresqlManager.java
View File

@ -33,16 +33,18 @@
import org.apache.hadoop.sqoop.util.ImportException;
/**
* Manages connections to Postgresql databases
* Manages connections to Postgresql databases.
*/
public class PostgresqlManager extends GenericJdbcManager {
public static final Log LOG = LogFactory.getLog(PostgresqlManager.class.getName());
public static final Log LOG = LogFactory.getLog(
PostgresqlManager.class.getName());
// driver class to ensure is loaded when making db connection.
private static final String DRIVER_CLASS = "org.postgresql.Driver";
private static final int POSTGRESQL_FETCH_SIZE = 50; // Fetch 50 rows at a time.
// Fetch 50 rows at a time.
private static final int POSTGRESQL_FETCH_SIZE = 50;
// set to true after we warn the user that we can use direct fastpath.
private static boolean warningPrinted = false;

32
src/java/org/apache/hadoop/sqoop/manager/SqlManager.java
View File

@ -62,7 +62,7 @@ public abstract class SqlManager extends ConnManager {
private Statement lastStatement;
/**
* Constructs the SqlManager
* Constructs the SqlManager.
* @param opts the SqoopOptions describing the user's requested action.
*/
public SqlManager(final SqoopOptions opts) {
@ -171,7 +171,8 @@ public Map<String, Integer> getColumnTypes(String tableName) {
}
@Override
public ResultSet readTable(String tableName, String[] columns) throws SQLException {
public ResultSet readTable(String tableName, String[] columns)
throws SQLException {
if (columns == null) {
columns = getColumnNames(tableName);
}
@ -212,7 +213,8 @@ public String[] listTables() {
DatabaseMetaData metaData = this.getConnection().getMetaData();
results = metaData.getTables(null, null, null, tableTypes);
} catch (SQLException sqlException) {
LOG.error("Error reading database metadata: " + sqlException.toString());
LOG.error("Error reading database metadata: "
+ sqlException.toString());
return null;
}
@ -264,25 +266,26 @@ public String getPrimaryKey(String tableName) {
getConnection().commit();
}
} catch (SQLException sqlException) {
LOG.error("Error reading primary key metadata: " + sqlException.toString());
LOG.error("Error reading primary key metadata: "
+ sqlException.toString());
return null;
}
}
/**
* Retrieve the actual connection from the outer ConnManager
* Retrieve the actual connection from the outer ConnManager.
*/
public abstract Connection getConnection() throws SQLException;
/**
* Determine what column to use to split the table.
* @param options the SqoopOptions controlling this import.
* @param opts the SqoopOptions controlling this import.
* @param tableName the table to import.
* @return the splitting column, if one is set or inferrable, or null
* otherwise.
*/
protected String getSplitColumn(SqoopOptions options, String tableName) {
String splitCol = options.getSplitByCol();
protected String getSplitColumn(SqoopOptions opts, String tableName) {
String splitCol = opts.getSplitByCol();
if (null == splitCol) {
// If the user didn't specify a splitting column, try to infer one.
splitCol = getPrimaryKey(tableName);
@ -344,7 +347,8 @@ protected ResultSet execute(String stmt, Object... args) throws SQLException {
* @return the name of a Java type to hold the sql datatype, or null if none.
*/
public String toJavaType(int sqlType) {
// mappings from http://java.sun.com/j2se/1.3/docs/guide/jdbc/getstart/mapping.html
// Mappings taken from:
// http://java.sun.com/j2se/1.3/docs/guide/jdbc/getstart/mapping.html
if (sqlType == Types.INTEGER) {
return "Integer";
} else if (sqlType == Types.VARCHAR) {
@ -396,7 +400,7 @@ public String toJavaType(int sqlType) {
}
/**
* Resolve a database-specific type to Hive data type
* Resolve a database-specific type to Hive data type.
* @param sqlType sql type
* @return hive type
*/
@ -489,7 +493,8 @@ protected Connection makeConnection() throws SQLException {
try {
Class.forName(driverClass);
} catch (ClassNotFoundException cnfe) {
throw new RuntimeException("Could not load db driver class: " + driverClass);
throw new RuntimeException("Could not load db driver class: "
+ driverClass);
}
String username = options.getUsername();
@ -497,7 +502,8 @@ protected Connection makeConnection() throws SQLException {
if (null == username) {
connection = DriverManager.getConnection(options.getConnectString());
} else {
connection = DriverManager.getConnection(options.getConnectString(), username, password);
connection = DriverManager.getConnection(options.getConnectString(),
username, password);
}
// We only use this for metadata queries. Loosest semantics are okay.
@ -508,7 +514,7 @@ protected Connection makeConnection() throws SQLException {
}
/**
* Export data stored in HDFS into a table in a database
* Export data stored in HDFS into a table in a database.
*/
public void exportTable(ExportJobContext context)
throws IOException, ExportException {

73
src/java/org/apache/hadoop/sqoop/mapreduce/AutoProgressMapper.java
View File

@ -34,27 +34,34 @@
public class AutoProgressMapper<KEYIN, VALIN, KEYOUT, VALOUT>
extends Mapper<KEYIN, VALIN, KEYOUT, VALOUT> {
public static final Log LOG = LogFactory.getLog(AutoProgressMapper.class.getName());
public static final Log LOG = LogFactory.getLog(
AutoProgressMapper.class.getName());
/** Total number of millis for which progress will be reported
by the auto-progress thread. If this is zero, then the auto-progress
thread will never voluntarily exit.
*/
/**
* Total number of millis for which progress will be reported by the
* auto-progress thread. If this is zero, then the auto-progress thread will
* never voluntarily exit.
*/
private int maxProgressPeriod;
/** Number of milliseconds to sleep for between loop iterations. Must be less
than report interval.
*/
/**
* Number of milliseconds to sleep for between loop iterations. Must be less
* than report interval.
*/
private int sleepInterval;
/** Number of milliseconds between calls to Reporter.progress(). Should be a multiple
of the sleepInterval.
*/
/**
* Number of milliseconds between calls to Reporter.progress().
* Should be a multiple of the sleepInterval.
*/
private int reportInterval;
public static final String MAX_PROGRESS_PERIOD_KEY = "sqoop.mapred.auto.progress.max";
public static final String SLEEP_INTERVAL_KEY = "sqoop.mapred.auto.progress.sleep";
public static final String REPORT_INTERVAL_KEY = "sqoop.mapred.auto.progress.report";
public static final String MAX_PROGRESS_PERIOD_KEY =
"sqoop.mapred.auto.progress.max";
public static final String SLEEP_INTERVAL_KEY =
"sqoop.mapred.auto.progress.sleep";
public static final String REPORT_INTERVAL_KEY =
"sqoop.mapred.auto.progress.report";
// Sleep for 10 seconds at a time.
static final int DEFAULT_SLEEP_INTERVAL = 10000;
@ -67,7 +74,7 @@ public class AutoProgressMapper<KEYIN, VALIN, KEYOUT, VALOUT>
private class ProgressThread extends Thread {
private volatile boolean keepGoing; // while this is true, thread runs.
private volatile boolean keepGoing; // While this is true, thread runs.
private Context context;
private long startTimeMillis;
@ -91,17 +98,20 @@ public void run() {
final long REPORT_INTERVAL = AutoProgressMapper.this.reportInterval;
final long SLEEP_INTERVAL = AutoProgressMapper.this.sleepInterval;
// in a loop:
// * Check that we haven't run for too long (maxProgressPeriod)
// * If it's been a report interval since we last made progress, make more.
// In a loop:
// * Check that we haven't run for too long (maxProgressPeriod).
// * If it's been a report interval since we last made progress,
// make more.
// * Sleep for a bit.
// * If the parent thread has signaled for exit, do so.
while (this.keepGoing) {
long curTimeMillis = System.currentTimeMillis();
if (MAX_PROGRESS != 0 && curTimeMillis - this.startTimeMillis > MAX_PROGRESS) {
if (MAX_PROGRESS != 0
&& curTimeMillis - this.startTimeMillis > MAX_PROGRESS) {
this.keepGoing = false;
LOG.info("Auto-progress thread exiting after " + MAX_PROGRESS + " ms.");
LOG.info("Auto-progress thread exiting after " + MAX_PROGRESS
+ " ms.");
break;
}
@ -130,23 +140,29 @@ public void run() {
/**
* Set configuration parameters for the auto-progress thread.
*/
private final void configureAutoProgress(Configuration job) {
this.maxProgressPeriod = job.getInt(MAX_PROGRESS_PERIOD_KEY, DEFAULT_MAX_PROGRESS);
this.sleepInterval = job.getInt(SLEEP_INTERVAL_KEY, DEFAULT_SLEEP_INTERVAL);
this.reportInterval = job.getInt(REPORT_INTERVAL_KEY, DEFAULT_REPORT_INTERVAL);
private void configureAutoProgress(Configuration job) {
this.maxProgressPeriod = job.getInt(MAX_PROGRESS_PERIOD_KEY,
DEFAULT_MAX_PROGRESS);
this.sleepInterval = job.getInt(SLEEP_INTERVAL_KEY,
DEFAULT_SLEEP_INTERVAL);
this.reportInterval = job.getInt(REPORT_INTERVAL_KEY,
DEFAULT_REPORT_INTERVAL);
if (this.reportInterval < 1) {
LOG.warn("Invalid " + REPORT_INTERVAL_KEY + "; setting to " + DEFAULT_REPORT_INTERVAL);
LOG.warn("Invalid " + REPORT_INTERVAL_KEY + "; setting to "
+ DEFAULT_REPORT_INTERVAL);
this.reportInterval = DEFAULT_REPORT_INTERVAL;
}
if (this.sleepInterval > this.reportInterval || this.sleepInterval < 1) {
LOG.warn("Invalid " + SLEEP_INTERVAL_KEY + "; setting to " + DEFAULT_SLEEP_INTERVAL);
LOG.warn("Invalid " + SLEEP_INTERVAL_KEY + "; setting to "
+ DEFAULT_SLEEP_INTERVAL);
this.sleepInterval = DEFAULT_SLEEP_INTERVAL;
}
if (this.maxProgressPeriod < 0) {
LOG.warn("Invalid " + MAX_PROGRESS_PERIOD_KEY + "; setting to " + DEFAULT_MAX_PROGRESS);
LOG.warn("Invalid " + MAX_PROGRESS_PERIOD_KEY + "; setting to "
+ DEFAULT_MAX_PROGRESS);
this.maxProgressPeriod = DEFAULT_MAX_PROGRESS;
}
}
@ -179,7 +195,8 @@ public void run(Context context) throws IOException, InterruptedException {
thread.join();
LOG.debug("Progress thread shutdown detected.");
} catch (InterruptedException ie) {
LOG.warn("Interrupted when waiting on auto-progress thread: " + ie.toString());
LOG.warn("Interrupted when waiting on auto-progress thread: "
+ ie.toString());
}
}
}

27
src/java/org/apache/hadoop/sqoop/mapreduce/DataDrivenImportJob.java
View File

@ -24,19 +24,12 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat;
@ -46,12 +39,8 @@
import org.apache.hadoop.sqoop.SqoopOptions;
import org.apache.hadoop.sqoop.manager.ConnManager;
import org.apache.hadoop.sqoop.lib.LargeObjectLoader;
import org.apache.hadoop.sqoop.orm.TableClassName;
import org.apache.hadoop.sqoop.shims.HadoopShim;
import org.apache.hadoop.sqoop.shims.ShimLoader;
import org.apache.hadoop.sqoop.util.ClassLoaderStack;
import org.apache.hadoop.sqoop.util.ImportException;
import org.apache.hadoop.sqoop.util.PerfCounters;
/**
* Actually runs a jdbc import job using the ORM files generated by the
@ -59,7 +48,8 @@
*/
public class DataDrivenImportJob extends ImportJobBase {
public static final Log LOG = LogFactory.getLog(DataDrivenImportJob.class.getName());
public static final Log LOG = LogFactory.getLog(
DataDrivenImportJob.class.getName());
@SuppressWarnings("unchecked")
public DataDrivenImportJob(final SqoopOptions opts) {
@ -115,11 +105,12 @@ protected void configureInputFormat(Job job, String tableName,
try {
String username = options.getUsername();
if (null == username || username.length() == 0) {
DBConfiguration.configureDB(job.getConfiguration(), mgr.getDriverClass(),
options.getConnectString());
DBConfiguration.configureDB(job.getConfiguration(),
mgr.getDriverClass(), options.getConnectString());
} else {
DBConfiguration.configureDB(job.getConfiguration(), mgr.getDriverClass(),
options.getConnectString(), username, options.getPassword());
DBConfiguration.configureDB(job.getConfiguration(),
mgr.getDriverClass(), options.getConnectString(),
username, options.getPassword());
}
String [] colNames = options.getColumns();
@ -139,8 +130,8 @@ protected void configureInputFormat(Job job, String tableName,
String whereClause = options.getWhereClause();
// We can't set the class properly in here, because we may not have the
// jar loaded in this JVM. So we start by calling setInput() with DBWritable
// and then overriding the string manually.
// jar loaded in this JVM. So we start by calling setInput() with
// DBWritable and then overriding the string manually.
DataDrivenDBInputFormat.setInput(job, DBWritable.class,
mgr.escapeTableName(tableName), whereClause,
mgr.escapeColName(splitByCol), sqlColNames);

8
src/java/org/apache/hadoop/sqoop/mapreduce/ExportJobBase.java
View File

@ -112,7 +112,8 @@ public static boolean isSequenceFiles(Configuration conf, Path p)
}
if (null == stat) {
LOG.warn("null FileStatus object in isSequenceFiles(); assuming false.");
LOG.warn("null FileStatus object in isSequenceFiles(); "
+ "assuming false.");
return false;
}
@ -239,7 +240,7 @@ protected boolean runJob(Job job) throws ClassNotFoundException, IOException,
/**
* Run an export job to dump a table from HDFS to a database
* Run an export job to dump a table from HDFS to a database.
* @throws IOException if the export job encounters an IO error
* @throws ExportException if the job fails unexpectedly or is misconfigured.
*/
@ -248,7 +249,8 @@ public void runExport() throws ExportException, IOException {
SqoopOptions options = context.getOptions();
Configuration conf = options.getConf();
String tableName = context.getTableName();
String tableClassName = new TableClassName(options).getClassForTable(tableName);
String tableClassName =
new TableClassName(options).getClassForTable(tableName);
String ormJarFile = context.getJarFile();
LOG.info("Beginning export of " + tableName);

17
src/java/org/apache/hadoop/sqoop/mapreduce/ImportJobBase.java
View File

@ -57,7 +57,8 @@
*/
public class ImportJobBase extends JobBase {
public static final Log LOG = LogFactory.getLog(ImportJobBase.class.getName());
public static final Log LOG = LogFactory.getLog(
ImportJobBase.class.getName());
public ImportJobBase() {
this(null);
@ -129,20 +130,24 @@ protected boolean runJob(Job job) throws ClassNotFoundException, IOException,
/**
* Run an import job to read a table in to HDFS
* Run an import job to read a table in to HDFS.
*
* @param tableName the database table to read
* @param ormJarFile the Jar file to insert into the dcache classpath. (may be null)
* @param splitByCol the column of the database table to use to split the import
* @param ormJarFile the Jar file to insert into the dcache classpath.
* (may be null)
* @param splitByCol the column of the database table to use to split
* the import
* @param conf A fresh Hadoop Configuration to use to build an MR job.
* @throws IOException if the job encountered an IO problem
* @throws ImportException if the job failed unexpectedly or was misconfigured.
* @throws ImportException if the job failed unexpectedly or was
* misconfigured.
*/
public void runImport(String tableName, String ormJarFile, String splitByCol,
Configuration conf) throws IOException, ImportException {
LOG.info("Beginning import of " + tableName);
String tableClassName = new TableClassName(options).getClassForTable(tableName);
String tableClassName =
new TableClassName(options).getClassForTable(tableName);
loadJars(conf, ormJarFile, tableClassName);
try {

3
src/java/org/apache/hadoop/sqoop/mapreduce/JdbcExportJob.java
View File

@ -35,10 +35,9 @@
import org.apache.hadoop.sqoop.ConnFactory;
import org.apache.hadoop.sqoop.manager.ConnManager;
import org.apache.hadoop.sqoop.manager.ExportJobContext;
import org.apache.hadoop.sqoop.shims.ShimLoader;
/**
* Run an export using JDBC (JDBC-based ExportOutputFormat)
* Run an export using JDBC (JDBC-based ExportOutputFormat).
*/
public class JdbcExportJob extends ExportJobBase {

16
src/java/org/apache/hadoop/sqoop/mapreduce/JobBase.java
View File

@ -26,7 +26,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.OutputFormat;
@ -92,23 +91,23 @@ protected Class<? extends OutputFormat> getOutputFormatClass()
return this.outputFormatClass;
}
/** Set the OutputFormat class to use for this job */
/** Set the OutputFormat class to use for this job. */
public void setOutputFormatClass(Class<? extends OutputFormat> cls) {
this.outputFormatClass = cls;
}
/** Set the InputFormat class to use for this job */
/** Set the InputFormat class to use for this job. */
public void setInputFormatClass(Class<? extends InputFormat> cls) {
this.inputFormatClass = cls;
}
/** Set the Mapper class to use for this job */
/** Set the Mapper class to use for this job. */
public void setMapperClass(Class<? extends Mapper> cls) {
this.mapperClass = cls;
}
/**
* Set the SqoopOptions configuring this job
* Set the SqoopOptions configuring this job.
*/
public void setOptions(SqoopOptions opts) {
this.options = opts;
@ -122,9 +121,10 @@ protected void loadJars(Configuration conf, String ormJarFile,
boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address"))
|| "local".equals(conf.get("mapred.job.tracker"));
if (isLocal) {
// If we're using the LocalJobRunner, then instead of using the compiled jar file
// as the job source, we're running in the current thread. Push on another classloader
// that loads from that jar in addition to everything currently on the classpath.
// If we're using the LocalJobRunner, then instead of using the compiled
// jar file as the job source, we're running in the current thread. Push
// on another classloader that loads from that jar in addition to
// everything currently on the classpath.
this.prevClassLoader = ClassLoaderStack.addJarFile(ormJarFile,
tableClassName);
}

27
src/java/org/apache/hadoop/sqoop/mapreduce/MySQLDumpImportJob.java
View File

@ -25,20 +25,10 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
@ -47,11 +37,7 @@
import org.apache.hadoop.sqoop.SqoopOptions;
import org.apache.hadoop.sqoop.manager.ConnManager;
import org.apache.hadoop.sqoop.manager.MySQLUtils;
import org.apache.hadoop.sqoop.orm.TableClassName;
import org.apache.hadoop.sqoop.shims.ShimLoader;
import org.apache.hadoop.sqoop.util.ClassLoaderStack;
import org.apache.hadoop.sqoop.util.ImportException;
import org.apache.hadoop.sqoop.util.PerfCounters;
/**
* Class that runs an import job using mysqldump in the mapper.
@ -82,11 +68,12 @@ protected void configureInputFormat(Job job, String tableName,
try {
String username = options.getUsername();
if (null == username || username.length() == 0) {
DBConfiguration.configureDB(job.getConfiguration(), mgr.getDriverClass(),
options.getConnectString());
DBConfiguration.configureDB(job.getConfiguration(),
mgr.getDriverClass(), options.getConnectString());
} else {
DBConfiguration.configureDB(job.getConfiguration(), mgr.getDriverClass(),
options.getConnectString(), username, options.getPassword());
DBConfiguration.configureDB(job.getConfiguration(),
mgr.getDriverClass(), options.getConnectString(), username,
options.getPassword());
}
String [] colNames = options.getColumns();
@ -106,8 +93,8 @@ protected void configureInputFormat(Job job, String tableName,
String whereClause = options.getWhereClause();
// We can't set the class properly in here, because we may not have the
// jar loaded in this JVM. So we start by calling setInput() with DBWritable
// and then overriding the string manually.
// jar loaded in this JVM. So we start by calling setInput() with
// DBWritable and then overriding the string manually.
// Note that mysqldump also does *not* want a quoted table name.
DataDrivenDBInputFormat.setInput(job, DBWritable.class,

53
src/java/org/apache/hadoop/sqoop/mapreduce/MySQLDumpMapper.java
View File

@ -19,14 +19,10 @@
package org.apache.hadoop.sqoop.mapreduce;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.CharBuffer;
import java.util.ArrayList;
import java.util.List;
@ -35,18 +31,13 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.sqoop.SqoopOptions;
import org.apache.hadoop.sqoop.lib.FieldFormatter;
import org.apache.hadoop.sqoop.lib.RecordParser;
import org.apache.hadoop.sqoop.manager.MySQLUtils;
import org.apache.hadoop.sqoop.util.AsyncSink;
import org.apache.hadoop.sqoop.util.DirectImportUtils;
import org.apache.hadoop.sqoop.util.ErrorableAsyncSink;
import org.apache.hadoop.sqoop.util.ErrorableThread;
import org.apache.hadoop.sqoop.util.ImportException;
import org.apache.hadoop.sqoop.util.JdbcUrl;
import org.apache.hadoop.sqoop.util.LoggingAsyncSink;
import org.apache.hadoop.sqoop.util.PerfCounters;
@ -57,7 +48,8 @@
public class MySQLDumpMapper
extends Mapper<String, NullWritable, String, NullWritable> {
public static final Log LOG = LogFactory.getLog(MySQLDumpMapper.class.getName());
public static final Log LOG = LogFactory.getLog(
MySQLDumpMapper.class.getName());
private Configuration conf;
@ -111,19 +103,22 @@ public void run() {
break; // EOF.
}
// this line is of the form "INSERT .. VALUES ( actual value text );"
// strip the leading preamble up to the '(' and the trailing ');'.
// this line is of the form "INSERT .. VALUES ( actual value text
// );" strip the leading preamble up to the '(' and the trailing
// ');'.
if (preambleLen == -1) {
// we haven't determined how long the preamble is. It's constant
// across all lines, so just figure this out once.
String recordStartMark = "VALUES (";
preambleLen = inLine.indexOf(recordStartMark) + recordStartMark.length();
preambleLen = inLine.indexOf(recordStartMark)
+ recordStartMark.length();
}
// chop off the leading and trailing text as we write the
// output to HDFS.
int len = inLine.length() - 2 - preambleLen;
context.write(inLine.substring(preambleLen, inLine.length() - 2), null);
context.write(inLine.substring(preambleLen, inLine.length() - 2),
null);
context.write("\n", null);
counters.addBytes(1 + len);
}
@ -235,18 +230,21 @@ public void run() {
break; // EOF.
}
// this line is of the form "INSERT .. VALUES ( actual value text );"
// strip the leading preamble up to the '(' and the trailing ');'.
// this line is of the form "INSERT .. VALUES ( actual value text
// );" strip the leading preamble up to the '(' and the trailing
// ');'.
if (preambleLen == -1) {
// we haven't determined how long the preamble is. It's constant
// across all lines, so just figure this out once.
String recordStartMark = "VALUES (";
preambleLen = inLine.indexOf(recordStartMark) + recordStartMark.length();
preambleLen = inLine.indexOf(recordStartMark)
+ recordStartMark.length();
}
// Wrap the input string in a char buffer that ignores the leading and trailing
// text.
CharBuffer charbuf = CharBuffer.wrap(inLine, preambleLen, inLine.length() - 2);
// Wrap the input string in a char buffer that ignores the leading
// and trailing text.
CharBuffer charbuf = CharBuffer.wrap(inLine, preambleLen,
inLine.length() - 2);
// Pass this along to the parser
List<String> fields = null;
@ -258,7 +256,8 @@ public void run() {
continue; // Skip emitting this row.
}
// For all of the output fields, emit them using the delimiters the user chooses.
// For all of the output fields, emit them using the delimiters
// the user chooses.
boolean first = true;
int recordLen = 1; // for the delimiter.
for (String field : fields) {
@ -312,10 +311,11 @@ public void map(String splitConditions, NullWritable val, Context context)
ArrayList<String> args = new ArrayList<String>();
String tableName = conf.get(MySQLUtils.TABLE_NAME_KEY);
// We need to parse the connect string URI to determine the database
// name. Using java.net.URL directly on the connect string will fail because
// Java doesn't respect arbitrary JDBC-based schemes. So we chop off the scheme
// (everything before '://') and replace it with 'http', which we know will work.
// We need to parse the connect string URI to determine the database name.
// Using java.net.URL directly on the connect string will fail because
// Java doesn't respect arbitrary JDBC-based schemes. So we chop off the
// scheme (everything before '://') and replace it with 'http', which we
// know will work.
String connectString = conf.get(MySQLUtils.CONNECT_STRING_KEY);
String databaseName = JdbcUrl.getDatabaseName(connectString);
String hostname = JdbcUrl.getHostName(connectString);
@ -391,7 +391,8 @@ public void map(String splitConditions, NullWritable val, Context context)
InputStream is = p.getInputStream();
if (MySQLUtils.outputDelimsAreMySQL(conf)) {
LOG.debug("Output delimiters conform to mysqldump; using straight copy");
LOG.debug("Output delimiters conform to mysqldump; "
+ "using straight copy");
sink = new CopyingAsyncSink(context, counters);
} else {
LOG.debug("User-specified delimiters; using reparsing import");

10
src/java/org/apache/hadoop/sqoop/mapreduce/MySQLExportJob.java
View File

@ -27,7 +27,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat;
@ -80,11 +79,12 @@ protected void configureInputFormat(Job job, String tableName,
mgr = new ConnFactory(conf).getManager(options);
String username = options.getUsername();
if (null == username || username.length() == 0) {
DBConfiguration.configureDB(job.getConfiguration(), mgr.getDriverClass(),
options.getConnectString());
DBConfiguration.configureDB(job.getConfiguration(),
mgr.getDriverClass(), options.getConnectString());
} else {
DBConfiguration.configureDB(job.getConfiguration(), mgr.getDriverClass(),
options.getConnectString(), username, options.getPassword());
DBConfiguration.configureDB(job.getConfiguration(),
mgr.getDriverClass(), options.getConnectString(), username,
options.getPassword());
}
String [] colNames = options.getColumns();

6
src/java/org/apache/hadoop/sqoop/mapreduce/MySQLExportMapper.java
View File

@ -30,7 +30,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.sqoop.lib.TaskId;
@ -53,7 +52,8 @@
public class MySQLExportMapper<KEYIN, VALIN>
extends Mapper<KEYIN, VALIN, NullWritable, NullWritable> {
public static final Log LOG = LogFactory.getLog(MySQLExportMapper.class.getName());
public static final Log LOG = LogFactory.getLog(
MySQLExportMapper.class.getName());
/** Configuration key that specifies the number of bytes before which it
* commits the current export transaction and opens a new one.
@ -82,7 +82,7 @@ public class MySQLExportMapper<KEYIN, VALIN>
protected AsyncSink outSink;
protected AsyncSink errSink;
/** File object where we wrote the user's password to pass to mysqlimport */
/** File object where we wrote the user's password to pass to mysqlimport. */
protected File passwordFile;
/** Character set used to write to mysqlimport. */

3
src/java/org/apache/hadoop/sqoop/mapreduce/SequenceFileExportMapper.java
View File

@ -31,7 +31,8 @@
* that DBWritable to the OutputFormat for writeback to the database.
*/
public class SequenceFileExportMapper
extends AutoProgressMapper<LongWritable, SqoopRecord, SqoopRecord, NullWritable> {
extends AutoProgressMapper<LongWritable, SqoopRecord, SqoopRecord,
NullWritable> {
public SequenceFileExportMapper() {
}

6
src/java/org/apache/hadoop/sqoop/mapreduce/SequenceFileImportMapper.java
View File

@ -31,12 +31,14 @@
* Imports records by writing them to a SequenceFile.
*/
public class SequenceFileImportMapper
extends AutoProgressMapper<LongWritable, SqoopRecord, LongWritable, SqoopRecord> {
extends AutoProgressMapper<LongWritable, SqoopRecord, LongWritable,
SqoopRecord> {
private LargeObjectLoader lobLoader;
@Override
protected void setup(Context context) throws IOException, InterruptedException {
protected void setup(Context context)
throws IOException, InterruptedException {
this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
FileOutputFormat.getWorkOutputPath(context));
}

9
src/java/org/apache/hadoop/sqoop/mapreduce/TextExportMapper.java
View File

@ -25,15 +25,15 @@
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.sqoop.lib.RecordParser;
import org.apache.hadoop.sqoop.lib.SqoopRecord;
/**
* Converts an input record from a string representation to a parsed Sqoop record
* and emits that DBWritable to the OutputFormat for writeback to the database.
* Converts an input record from a string representation to a parsed Sqoop
* record and emits that DBWritable to the OutputFormat for writeback to the
* database.
*/
public class TextExportMapper
extends AutoProgressMapper<LongWritable, Text, SqoopRecord, NullWritable> {
@ -67,7 +67,8 @@ protected void setup(Context context)
}
if (null == recordImpl) {
throw new IOException("Could not instantiate object of type " + recordClassName);
throw new IOException("Could not instantiate object of type "
+ recordClassName);
}
}

3
src/java/org/apache/hadoop/sqoop/mapreduce/TextImportMapper.java
View File

@ -43,7 +43,8 @@ public TextImportMapper() {
}
@Override
protected void setup(Context context) throws IOException, InterruptedException {
protected void setup(Context context)
throws IOException, InterruptedException {
this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
FileOutputFormat.getWorkOutputPath(context));
}

261
src/java/org/apache/hadoop/sqoop/orm/ClassWriter.java
View File

@ -21,7 +21,6 @@
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.sqoop.SqoopOptions;
import org.apache.hadoop.sqoop.manager.ConnManager;
import org.apache.hadoop.sqoop.manager.SqlManager;
import org.apache.hadoop.sqoop.lib.BigDecimalSerializer;
import org.apache.hadoop.sqoop.lib.FieldFormatter;
import org.apache.hadoop.sqoop.lib.JdbcWritableBridge;
@ -45,7 +44,7 @@
import org.apache.commons.logging.LogFactory;
/**
* Creates an ORM class to represent a table from a database
* Creates an ORM class to represent a table from a database.
*/
public class ClassWriter {
@ -209,12 +208,13 @@ public static String toIdentifier(String candidate) {
/**
* @param javaType
* @return the name of the method of JdbcWritableBridge to read an entry with a given java type.
* @return the name of the method of JdbcWritableBridge to read an entry
* with a given java type.
*/
private String dbGetterForType(String javaType) {
// All Class-based types (e.g., java.math.BigDecimal) are handled with
// "readBar" where some.package.foo.Bar is the canonical class name.
// Turn the javaType string into the getter type string.
// "readBar" where some.package.foo.Bar is the canonical class name. Turn
// the javaType string into the getter type string.
String [] parts = javaType.split("\\.");
if (parts.length == 0) {
@ -224,18 +224,21 @@ private String dbGetterForType(String javaType) {
String lastPart = parts[parts.length - 1];
try {
String getter = "read" + Character.toUpperCase(lastPart.charAt(0)) + lastPart.substring(1);
String getter = "read" + Character.toUpperCase(lastPart.charAt(0))
+ lastPart.substring(1);
return getter;
} catch (StringIndexOutOfBoundsException oob) {
// lastPart.*() doesn't work on empty strings.
LOG.error("Could not infer JdbcWritableBridge getter for Java type " + javaType);
LOG.error("Could not infer JdbcWritableBridge getter for Java type "
+ javaType);
return null;
}
}
/**
* @param javaType
* @return the name of the method of JdbcWritableBridge to write an entry with a given java type.
* @return the name of the method of JdbcWritableBridge to write an entry
* with a given java type.
*/
private String dbSetterForType(String javaType) {
// TODO(aaron): Lots of unit tests needed here.
@ -249,11 +252,13 @@ private String dbSetterForType(String javaType) {
String lastPart = parts[parts.length - 1];
try {
String setter = "write" + Character.toUpperCase(lastPart.charAt(0)) + lastPart.substring(1);
String setter = "write" + Character.toUpperCase(lastPart.charAt(0))
+ lastPart.substring(1);
return setter;
} catch (StringIndexOutOfBoundsException oob) {
// lastPart.*() doesn't work on empty strings.
LOG.error("Could not infer PreparedStatement setter for Java type " + javaType);
LOG.error("Could not infer PreparedStatement setter for Java type "
+ javaType);
return null;
}
}
@ -262,7 +267,7 @@ private String stringifierForType(String javaType, String colName) {
if (javaType.equals("String")) {
return colName;
} else {
// this is an object type -- just call its toString() in a null-safe way.
// This is an object type -- just call its toString() in a null-safe way.
return "\"\" + " + colName;
}
}
@ -271,36 +276,49 @@ private String stringifierForType(String javaType, String colName) {
* @param javaType the type to read
* @param inputObj the name of the DataInput to read from
* @param colName the column name to read
* @return the line of code involving a DataInput object to read an entry with a given java type.
* @return the line of code involving a DataInput object to read an entry
* with a given java type.
*/
private String rpcGetterForType(String javaType, String inputObj, String colName) {
private String rpcGetterForType(String javaType, String inputObj,
String colName) {
if (javaType.equals("Integer")) {
return " this." + colName + " = Integer.valueOf(" + inputObj + ".readInt());\n";
return " this." + colName + " = Integer.valueOf(" + inputObj
+ ".readInt());\n";
} else if (javaType.equals("Long")) {
return " this." + colName + " = Long.valueOf(" + inputObj + ".readLong());\n";
return " this." + colName + " = Long.valueOf(" + inputObj
+ ".readLong());\n";
} else if (javaType.equals("Float")) {
return " this." + colName + " = Float.valueOf(" + inputObj + ".readFloat());\n";
return " this." + colName + " = Float.valueOf(" + inputObj
+ ".readFloat());\n";
} else if (javaType.equals("Double")) {
return " this." + colName + " = Double.valueOf(" + inputObj + ".readDouble());\n";
return " this." + colName + " = Double.valueOf(" + inputObj
+ ".readDouble());\n";
} else if (javaType.equals("Boolean")) {
return " this." + colName + " = Boolean.valueOf(" + inputObj + ".readBoolean());\n";
return " this." + colName + " = Boolean.valueOf(" + inputObj
+ ".readBoolean());\n";
} else if (javaType.equals("String")) {
return " this." + colName + " = Text.readString(" + inputObj + ");\n";
} else if (javaType.equals("java.sql.Date")) {
return " this." + colName + " = new Date(" + inputObj + ".readLong());\n";
return " this." + colName + " = new Date(" + inputObj
+ ".readLong());\n";
} else if (javaType.equals("java.sql.Time")) {
return " this." + colName + " = new Time(" + inputObj + ".readLong());\n";
return " this." + colName + " = new Time(" + inputObj
+ ".readLong());\n";
} else if (javaType.equals("java.sql.Timestamp")) {
return " this." + colName + " = new Timestamp(" + inputObj + ".readLong());\n"
+ " this." + colName + ".setNanos(" + inputObj + ".readInt());\n";
return " this." + colName + " = new Timestamp(" + inputObj
+ ".readLong());\n" + " this." + colName + ".setNanos("
+ inputObj + ".readInt());\n";
} else if (javaType.equals("java.math.BigDecimal")) {
return " this." + colName + " = " + BigDecimalSerializer.class.getCanonicalName()
return " this." + colName + " = "
+ BigDecimalSerializer.class.getCanonicalName()
+ ".readFields(" + inputObj + ");\n";
} else if (javaType.equals(ClobRef.class.getName())) {
return " this." + colName + " = " + LobSerializer.class.getCanonicalName()
return " this." + colName + " = "
+ LobSerializer.class.getCanonicalName()
+ ".readClobFields(" + inputObj + ");\n";
} else if (javaType.equals(BlobRef.class.getName())) {
return " this." + colName + " = " + LobSerializer.class.getCanonicalName()
return " this." + colName + " = "
+ LobSerializer.class.getCanonicalName()
+ ".readBlobFields(" + inputObj + ");\n";
} else if (javaType.equals(BytesWritable.class.getName())) {
return " this." + colName + " = new BytesWritable();\n"
@ -312,13 +330,14 @@ private String rpcGetterForType(String javaType, String inputObj, String colName
}
/**
* Deserialize a possibly-null value from the DataInput stream
* Deserialize a possibly-null value from the DataInput stream.
* @param javaType name of the type to deserialize if it's not null.
* @param inputObj name of the DataInput to read from
* @param colName the column name to read.
* @return
*/
private String rpcGetterForMaybeNull(String javaType, String inputObj, String colName) {
private String rpcGetterForMaybeNull(String javaType, String inputObj,
String colName) {
return " if (" + inputObj + ".readBoolean()) { \n"
+ " this." + colName + " = null;\n"
+ " } else {\n"
@ -330,10 +349,11 @@ private String rpcGetterForMaybeNull(String javaType, String inputObj, String co
* @param javaType the type to write
* @param inputObj the name of the DataOutput to write to
* @param colName the column name to write
* @return the line of code involving a DataOutput object to write an entry with
* a given java type.
* @return the line of code involving a DataOutput object to write an entry
* with a given java type.
*/
private String rpcSetterForType(String javaType, String outputObj, String colName) {
private String rpcSetterForType(String javaType, String outputObj,
String colName) {
if (javaType.equals("Integer")) {
return " " + outputObj + ".writeInt(this." + colName + ");\n";
} else if (javaType.equals("Long")) {
@ -347,12 +367,15 @@ private String rpcSetterForType(String javaType, String outputObj, String colNam
} else if (javaType.equals("String")) {
return " Text.writeString(" + outputObj + ", " + colName + ");\n";
} else if (javaType.equals("java.sql.Date")) {
return " " + outputObj + ".writeLong(this." + colName + ".getTime());\n";
return " " + outputObj + ".writeLong(this." + colName
+ ".getTime());\n";
} else if (javaType.equals("java.sql.Time")) {
return " " + outputObj + ".writeLong(this." + colName + ".getTime());\n";
return " " + outputObj + ".writeLong(this." + colName
+ ".getTime());\n";
} else if (javaType.equals("java.sql.Timestamp")) {
return " " + outputObj + ".writeLong(this." + colName + ".getTime());\n"
+ " " + outputObj + ".writeInt(this." + colName + ".getNanos());\n";
return " " + outputObj + ".writeLong(this." + colName
+ ".getTime());\n" + " " + outputObj + ".writeInt(this." + colName
+ ".getNanos());\n";
} else if (javaType.equals(BytesWritable.class.getName())) {
return " this." + colName + ".write(" + outputObj + ");\n";
} else if (javaType.equals("java.math.BigDecimal")) {
@ -378,7 +401,8 @@ private String rpcSetterForType(String javaType, String outputObj, String colNam
* @param colName the column name to read.
* @return
*/
private String rpcSetterForMaybeNull(String javaType, String outputObj, String colName) {
private String rpcSetterForMaybeNull(String javaType, String outputObj,
String colName) {
return " if (null == this." + colName + ") { \n"
+ " " + outputObj + ".writeBoolean(true);\n"
+ " } else {\n"
@ -388,13 +412,13 @@ private String rpcSetterForMaybeNull(String javaType, String outputObj, String c
}
/**
* Generate a member field and getter method for each column
* Generate a member field and getter method for each column.
* @param columnTypes - mapping from column names to sql types
* @param colNames - ordered list of column names for table.
* @param sb - StringBuilder to append code to
*/
private void generateFields(Map<String, Integer> columnTypes, String [] colNames,
StringBuilder sb) {
private void generateFields(Map<String, Integer> columnTypes,
String [] colNames, StringBuilder sb) {
for (String col : colNames) {
int sqlType = columnTypes.get(col);
@ -412,15 +436,16 @@ private void generateFields(Map<String, Integer> columnTypes, String [] colNames
}
/**
* Generate the readFields() method used by the database
* Generate the readFields() method used by the database.
* @param columnTypes - mapping from column names to sql types
* @param colNames - ordered list of column names for table.
* @param sb - StringBuilder to append code to
*/
private void generateDbRead(Map<String, Integer> columnTypes, String [] colNames,
StringBuilder sb) {
private void generateDbRead(Map<String, Integer> columnTypes,
String [] colNames, StringBuilder sb) {
sb.append(" public void readFields(ResultSet __dbResults) throws SQLException {\n");
sb.append(" public void readFields(ResultSet __dbResults) ");
sb.append("throws SQLException {\n");
// Save ResultSet object cursor for use in LargeObjectLoader
// if necessary.
@ -462,7 +487,8 @@ private void generateLoadLargeObjects(Map<String, Integer> columnTypes,
// readFields() method generated by generateDbRead().
sb.append(" public void loadLargeObjects(LargeObjectLoader __loader)\n");
sb.append(" throws SQLException, IOException, InterruptedException {\n");
sb.append(" throws SQLException, IOException, ");
sb.append("InterruptedException {\n");
int fieldNum = 0;
@ -479,9 +505,9 @@ private void generateLoadLargeObjects(Map<String, Integer> columnTypes,
String getterMethod = dbGetterForType(javaType);
if ("readClobRef".equals(getterMethod)
|| "readBlobRef".equals(getterMethod)) {
// This field is a blob/clob field with delayed loading.
// Call the appropriate LargeObjectLoader method (which has the
// same name as a JdbcWritableBridge method).
// This field is a blob/clob field with delayed loading. Call the
// appropriate LargeObjectLoader method (which has the same name as a
// JdbcWritableBridge method).
sb.append(" this." + col + " = __loader." + getterMethod
+ "(" + fieldNum + ", this.__cur_result_set);\n");
}
@ -491,19 +517,21 @@ private void generateLoadLargeObjects(Map<String, Integer> columnTypes,
/**
* Generate the write() method used by the database
* Generate the write() method used by the database.
* @param columnTypes - mapping from column names to sql types
* @param colNames - ordered list of column names for table.
* @param sb - StringBuilder to append code to
*/
private void generateDbWrite(Map<String, Integer> columnTypes, String [] colNames,
StringBuilder sb) {
private void generateDbWrite(Map<String, Integer> columnTypes,
String [] colNames, StringBuilder sb) {
sb.append(" public void write(PreparedStatement __dbStmt) throws SQLException {\n");
sb.append(" public void write(PreparedStatement __dbStmt) "
+ "throws SQLException {\n");
sb.append(" write(__dbStmt, 0);\n");
sb.append(" }\n\n");
sb.append(" public int write(PreparedStatement __dbStmt, int __off) throws SQLException {\n");
sb.append(" public int write(PreparedStatement __dbStmt, int __off) "
+ "throws SQLException {\n");
int fieldNum = 0;
@ -533,15 +561,16 @@ private void generateDbWrite(Map<String, Integer> columnTypes, String [] colName
/**
* Generate the readFields() method used by the Hadoop RPC system
* Generate the readFields() method used by the Hadoop RPC system.
* @param columnTypes - mapping from column names to sql types
* @param colNames - ordered list of column names for table.
* @param sb - StringBuilder to append code to
*/
private void generateHadoopRead(Map<String, Integer> columnTypes, String [] colNames,
StringBuilder sb) {
private void generateHadoopRead(Map<String, Integer> columnTypes,
String [] colNames, StringBuilder sb) {
sb.append(" public void readFields(DataInput __dataIn) throws IOException {\n");
sb.append(" public void readFields(DataInput __dataIn) "
+ "throws IOException {\n");
for (String col : colNames) {
int sqlType = columnTypes.get(col);
@ -610,27 +639,27 @@ private void generateCloneMethod(Map<String, Integer> columnTypes,
* @param colNames - ordered list of column names for table.
* @param sb - StringBuilder to append code to
*/
private void generateToString(Map<String, Integer> columnTypes, String [] colNames,
StringBuilder sb) {
private void generateToString(Map<String, Integer> columnTypes,
String [] colNames, StringBuilder sb) {
// Embed the delimiters into the class, as characters...
sb.append(" private static final char __OUTPUT_FIELD_DELIM_CHAR = " +
sb.append(" private static final char __OUTPUT_FIELD_DELIM_CHAR = "
+ (int)options.getOutputFieldDelim() + ";\n");
sb.append(" private static final char __OUTPUT_RECORD_DELIM_CHAR = "
sb.append(" private static final char __OUTPUT_RECORD_DELIM_CHAR = "
+ (int)options.getOutputRecordDelim() + ";\n");
// as strings...
sb.append(" private static final String __OUTPUT_FIELD_DELIM = \"\" + (char) "
+ (int) options.getOutputFieldDelim() + ";\n");
sb.append(" private static final String __OUTPUT_RECORD_DELIM = \"\" + (char) "
+ (int) options.getOutputRecordDelim() + ";\n");
sb.append(" private static final String __OUTPUT_ENCLOSED_BY = \"\" + (char) "
+ (int) options.getOutputEnclosedBy() + ";\n");
sb.append(" private static final String __OUTPUT_ESCAPED_BY = \"\" + (char) "
+ (int) options.getOutputEscapedBy() + ";\n");
sb.append(" private static final String __OUTPUT_FIELD_DELIM = "
+ "\"\" + (char) " + (int) options.getOutputFieldDelim() + ";\n");
sb.append(" private static final String __OUTPUT_RECORD_DELIM = "
+ "\"\" + (char) " + (int) options.getOutputRecordDelim() + ";\n");
sb.append(" private static final String __OUTPUT_ENCLOSED_BY = "
+ "\"\" + (char) " + (int) options.getOutputEnclosedBy() + ";\n");
sb.append(" private static final String __OUTPUT_ESCAPED_BY = "
+ "\"\" + (char) " + (int) options.getOutputEscapedBy() + ";\n");
// and some more options.
sb.append(" private static final boolean __OUTPUT_ENCLOSE_REQUIRED = "
sb.append(" private static final boolean __OUTPUT_ENCLOSE_REQUIRED = "
+ options.isOutputEncloseRequired() + ";\n");
sb.append(" private static final char [] __OUTPUT_DELIMITER_LIST = { "
+ "__OUTPUT_FIELD_DELIM_CHAR, __OUTPUT_RECORD_DELIM_CHAR };\n\n");
@ -662,8 +691,8 @@ private void generateToString(Map<String, Integer> columnTypes, String [] colNam
}
sb.append(" __sb.append(FieldFormatter.escapeAndEnclose(" + stringExpr
+ ", __OUTPUT_ESCAPED_BY, __OUTPUT_ENCLOSED_BY, __OUTPUT_DELIMITER_LIST, "
+ "__OUTPUT_ENCLOSE_REQUIRED));\n");
+ ", __OUTPUT_ESCAPED_BY, __OUTPUT_ENCLOSED_BY, "
+ "__OUTPUT_DELIMITER_LIST, __OUTPUT_ENCLOSE_REQUIRED));\n");
}
@ -675,17 +704,21 @@ private void generateToString(Map<String, Integer> columnTypes, String [] colNam
/**
* Helper method for generateParser(). Writes out the parse() method for one particular
* type we support as an input string-ish type.
* Helper method for generateParser(). Writes out the parse() method for one
* particular type we support as an input string-ish type.
*/
private void generateParseMethod(String typ, StringBuilder sb) {
sb.append(" public void parse(" + typ + " __record) throws RecordParser.ParseError {\n");
sb.append(" public void parse(" + typ + " __record) "
+ "throws RecordParser.ParseError {\n");
sb.append(" if (null == this.__parser) {\n");
sb.append(" this.__parser = new RecordParser(__INPUT_FIELD_DELIM_CHAR, ");
sb.append("__INPUT_RECORD_DELIM_CHAR, __INPUT_ENCLOSED_BY_CHAR, __INPUT_ESCAPED_BY_CHAR, ");
sb.append(" this.__parser = new RecordParser("
+ "__INPUT_FIELD_DELIM_CHAR, ");
sb.append("__INPUT_RECORD_DELIM_CHAR, __INPUT_ENCLOSED_BY_CHAR, "
+ "__INPUT_ESCAPED_BY_CHAR, ");
sb.append("__INPUT_ENCLOSE_REQUIRED);\n");
sb.append(" }\n");
sb.append(" List<String> __fields = this.__parser.parseRecord(__record);\n");
sb.append(" List<String> __fields = "
+ "this.__parser.parseRecord(__record);\n");
sb.append(" __loadFromFields(__fields);\n");
sb.append(" }\n\n");
}
@ -701,18 +734,20 @@ private void parseNullVal(String colName, StringBuilder sb) {
}
/**
* Helper method for generateParser(). Generates the code that loads one field of
* a specified name and type from the next element of the field strings list.
* Helper method for generateParser(). Generates the code that loads one
* field of a specified name and type from the next element of the field
* strings list.
*/
private void parseColumn(String colName, int colType, StringBuilder sb) {
// assume that we have __it and __cur_str vars, based on __loadFromFields() code.
// assume that we have __it and __cur_str vars, based on
// __loadFromFields() code.
sb.append(" __cur_str = __it.next();\n");
String javaType = connManager.toJavaType(colType);
parseNullVal(colName, sb);
if (javaType.equals("String")) {
// TODO(aaron): Distinguish between 'null' and null. Currently they both set the
// actual object to null.
// TODO(aaron): Distinguish between 'null' and null. Currently they both
// set the actual object to null.
sb.append(" this." + colName + " = __cur_str;\n");
} else if (javaType.equals("Integer")) {
sb.append(" this." + colName + " = Integer.valueOf(__cur_str);\n");
@ -725,13 +760,17 @@ private void parseColumn(String colName, int colType, StringBuilder sb) {
} else if (javaType.equals("Boolean")) {
sb.append(" this." + colName + " = Boolean.valueOf(__cur_str);\n");
} else if (javaType.equals("java.sql.Date")) {
sb.append(" this." + colName + " = java.sql.Date.valueOf(__cur_str);\n");
sb.append(" this." + colName
+ " = java.sql.Date.valueOf(__cur_str);\n");
} else if (javaType.equals("java.sql.Time")) {
sb.append(" this." + colName + " = java.sql.Time.valueOf(__cur_str);\n");
sb.append(" this." + colName
+ " = java.sql.Time.valueOf(__cur_str);\n");
} else if (javaType.equals("java.sql.Timestamp")) {
sb.append(" this." + colName + " = java.sql.Timestamp.valueOf(__cur_str);\n");
sb.append(" this." + colName
+ " = java.sql.Timestamp.valueOf(__cur_str);\n");
} else if (javaType.equals("java.math.BigDecimal")) {
sb.append(" this." + colName + " = new java.math.BigDecimal(__cur_str);\n");
sb.append(" this." + colName
+ " = new java.math.BigDecimal(__cur_str);\n");
} else if (javaType.equals(ClobRef.class.getName())) {
sb.append(" this." + colName + " = ClobRef.parse(__cur_str);\n");
} else if (javaType.equals(BlobRef.class.getName())) {
@ -744,18 +783,19 @@ private void parseColumn(String colName, int colType, StringBuilder sb) {
}
/**
* Generate the parse() method
* Generate the parse() method.
* @param columnTypes - mapping from column names to sql types
* @param colNames - ordered list of column names for table.
* @param sb - StringBuilder to append code to
*/
private void generateParser(Map<String, Integer> columnTypes, String [] colNames,
StringBuilder sb) {
private void generateParser(Map<String, Integer> columnTypes,
String [] colNames, StringBuilder sb) {
// Embed into the class the delimiter characters to use when parsing input records.
// Note that these can differ from the delims to use as output via toString(), if
// the user wants to use this class to convert one format to another.
sb.append(" private static final char __INPUT_FIELD_DELIM_CHAR = " +
// Embed into the class the delimiter characters to use when parsing input
// records. Note that these can differ from the delims to use as output
// via toString(), if the user wants to use this class to convert one
// format to another.
sb.append(" private static final char __INPUT_FIELD_DELIM_CHAR = "
+ (int)options.getInputFieldDelim() + ";\n");
sb.append(" private static final char __INPUT_RECORD_DELIM_CHAR = "
+ (int)options.getInputRecordDelim() + ";\n");
@ -778,9 +818,9 @@ private void generateParser(Map<String, Integer> columnTypes, String [] colNames
generateParseMethod("ByteBuffer", sb);
generateParseMethod("CharBuffer", sb);
// The wrapper methods call __loadFromFields() to actually interpret the raw
// field data as string, int, boolean, etc. The generation of this method is
// type-dependent for the fields.
// The wrapper methods call __loadFromFields() to actually interpret the
// raw field data as string, int, boolean, etc. The generation of this
// method is type-dependent for the fields.
sb.append(" private void __loadFromFields(List<String> fields) {\n");
sb.append(" Iterator<String> __it = fields.listIterator();\n");
sb.append(" String __cur_str;\n");
@ -792,15 +832,16 @@ private void generateParser(Map<String, Integer> columnTypes, String [] colNames
}
/**
* Generate the write() method used by the Hadoop RPC system
* Generate the write() method used by the Hadoop RPC system.
* @param columnTypes - mapping from column names to sql types
* @param colNames - ordered list of column names for table.
* @param sb - StringBuilder to append code to
*/
private void generateHadoopWrite(Map<String, Integer> columnTypes, String [] colNames,
StringBuilder sb) {
private void generateHadoopWrite(Map<String, Integer> columnTypes,
String [] colNames, StringBuilder sb) {
sb.append(" public void write(DataOutput __dataOut) throws IOException {\n");
sb.append(" public void write(DataOutput __dataOut) "
+ "throws IOException {\n");
for (String col : colNames) {
int sqlType = columnTypes.get(col);
@ -840,21 +881,22 @@ public void generate() throws IOException {
String identifier = toIdentifier(col);
cleanedColNames[i] = identifier;
// make sure the col->type mapping holds for the
// Make sure the col->type mapping holds for the
// new identifier name, too.
columnTypes.put(identifier, columnTypes.get(col));
}
// Generate the Java code
// Generate the Java code.
StringBuilder sb = generateClassForColumns(columnTypes, cleanedColNames);
// Write this out to a file.
String codeOutDir = options.getCodeOutputDir();
// Get the class name to generate, which includes package components
// Get the class name to generate, which includes package components.
String className = new TableClassName(options).getClassForTable(tableName);
// convert the '.' characters to '/' characters
String sourceFilename = className.replace('.', File.separatorChar) + ".java";
// Convert the '.' characters to '/' characters.
String sourceFilename = className.replace('.', File.separatorChar)
+ ".java";
String filename = codeOutDir + sourceFilename;
if (LOG.isDebugEnabled()) {
@ -908,7 +950,7 @@ public void generate() throws IOException {
}
/**
* Generate the ORM code for a table object containing the named columns
* Generate the ORM code for a table object containing the named columns.
* @param columnTypes - mapping from column names to sql types
* @param colNames - ordered list of column names for table.
* @return - A StringBuilder that contains the text of the class code.
@ -917,7 +959,8 @@ public StringBuilder generateClassForColumns(Map<String, Integer> columnTypes,
String [] colNames) {
StringBuilder sb = new StringBuilder();
sb.append("// ORM class for " + tableName + "\n");
sb.append("// WARNING: This class is AUTO-GENERATED. Modify at your own risk.\n");
sb.append("// WARNING: This class is AUTO-GENERATED. "
+ "Modify at your own risk.\n");
TableClassName tableNameInfo = new TableClassName(options);
@ -958,7 +1001,8 @@ public StringBuilder generateClassForColumns(Map<String, Integer> columnTypes,
String className = tableNameInfo.getShortClassForTable(tableName);
sb.append("public class " + className
+ " implements DBWritable, SqoopRecord, Writable {\n");
sb.append(" public static final int PROTOCOL_VERSION = " + CLASS_WRITER_VERSION + ";\n");
sb.append(" public static final int PROTOCOL_VERSION = "
+ CLASS_WRITER_VERSION + ";\n");
sb.append(" protected ResultSet __cur_result_set;\n");
generateFields(columnTypes, colNames, sb);
generateDbRead(columnTypes, colNames, sb);
@ -970,7 +1014,8 @@ public StringBuilder generateClassForColumns(Map<String, Integer> columnTypes,
generateParser(columnTypes, colNames, sb);
generateCloneMethod(columnTypes, colNames, sb);
// TODO(aaron): Generate hashCode(), compareTo(), equals() so it can be a WritableComparable
// TODO(aaron): Generate hashCode(), compareTo(), equals() so it can be a
// WritableComparable
sb.append("}\n");

41
src/java/org/apache/hadoop/sqoop/orm/CompilationManager.java
View File

@ -38,7 +38,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.sqoop.SqoopOptions;
@ -49,15 +48,13 @@
* Manages the compilation of a bunch of .java files into .class files
* and eventually a jar.
*
* Also embeds this program's jar into the lib/ directory inside the compiled jar
* to ensure that the job runs correctly.
*
*
*
* Also embeds this program's jar into the lib/ directory inside the compiled
* jar to ensure that the job runs correctly.
*/
public class CompilationManager {
public static final Log LOG = LogFactory.getLog(CompilationManager.class.getName());
public static final Log LOG = LogFactory.getLog(
CompilationManager.class.getName());
private SqoopOptions options;
private List<String> sources;
@ -98,7 +95,8 @@ private String findHadoopCoreJar() {
}
for (File f : entries) {
if (f.getName().startsWith("hadoop-") && f.getName().endsWith("-core.jar")) {
if (f.getName().startsWith("hadoop-")
&& f.getName().endsWith("-core.jar")) {
LOG.info("Found hadoop core jar at: " + f.getAbsolutePath());
return f.getAbsolutePath();
}
@ -128,9 +126,9 @@ public void compile() throws IOException {
// find hadoop-*-core.jar for classpath.
String coreJar = findHadoopCoreJar();
if (null == coreJar) {
// Couldn't find a core jar to insert into the CP for compilation.
// If, however, we're running this from a unit test, then the path
// to the .class files might be set via the hadoop.alt.classpath property
// Couldn't find a core jar to insert into the CP for compilation. If,
// however, we're running this from a unit test, then the path to the
// .class files might be set via the hadoop.alt.classpath property
// instead. Check there first.
String coreClassesPath = System.getProperty("hadoop.alt.classpath");
if (null == coreClassesPath) {
@ -200,7 +198,7 @@ public void compile() throws IOException {
}
/**
* @return the complete filename of the .jar file to generate */
* @return the complete filename of the .jar file to generate. */
public String getJarFilename() {
String jarOutDir = options.getJarOutputDir();
String tableName = options.getTableName();
@ -235,11 +233,11 @@ private void addClassFilesFromDir(File dir, JarOutputStream jstream)
baseDirName = baseDirName + File.separator;
}
// for each input class file, create a zipfile entry for it,
// For each input class file, create a zipfile entry for it,
// read the file into a buffer, and write it to the jar file.
for (File entry : dirEntries) {
if (!entry.isDirectory()) {
// chomp off the portion of the full path that is shared
// Chomp off the portion of the full path that is shared
// with the base directory where class files were put;
// we only record the subdir parts in the zip entry.
String fullPath = entry.getAbsolutePath();
@ -247,7 +245,8 @@ private void addClassFilesFromDir(File dir, JarOutputStream jstream)
boolean include = chompedPath.endsWith(".class")
&& sources.contains(
chompedPath.substring(0, chompedPath.length() - ".class".length()) + ".java");
chompedPath.substring(0, chompedPath.length() - ".class".length())
+ ".java");
if (include) {
// include this file.
@ -262,7 +261,7 @@ private void addClassFilesFromDir(File dir, JarOutputStream jstream)
}
/**
* Create an output jar file to use when executing MapReduce jobs
* Create an output jar file to use when executing MapReduce jobs.
*/
public void jar() throws IOException {
String jarOutDir = options.getJarOutputDir();
@ -293,7 +292,8 @@ public void jar() throws IOException {
addLibJar(thisJarFile, jstream);
} else {
// couldn't find our own jar (we were running from .class files?)
LOG.warn("Could not find jar for Sqoop; MapReduce jobs may not run correctly.");
LOG.warn("Could not find jar for Sqoop; "
+ "MapReduce jobs may not run correctly.");
}
String shimJarFile = findShimJar();
@ -347,12 +347,13 @@ private void addLibJar(String jarFilename, JarOutputStream jstream)
private static final int BUFFER_SZ = 4096;
/**
* utility method to copy a .class file into the jar stream.
* Utility method to copy a .class file into the jar stream.
* @param f
* @param ostream
* @throws IOException
*/
private void copyFileToStream(File f, OutputStream ostream) throws IOException {
private void copyFileToStream(File f, OutputStream ostream)
throws IOException {
FileInputStream fis = new FileInputStream(f);
byte [] buffer = new byte[BUFFER_SZ];
try {
@ -381,7 +382,7 @@ private String findShimJar() {
return findJarForClass(h.getClass());
}
// method mostly cloned from o.a.h.mapred.JobConf.findContainingJar()
// Method mostly cloned from o.a.h.mapred.JobConf.findContainingJar().
private String findJarForClass(Class<? extends Object> classObj) {
ClassLoader loader = classObj.getClassLoader();
String classFile = classObj.getName().replaceAll("\\.", "/") + ".class";

26
src/java/org/apache/hadoop/sqoop/orm/TableClassName.java
View File

@ -25,18 +25,20 @@
/**
* Reconciles the table name being imported with the class naming information
* specified in SqoopOptions to determine the actual package and class name
* to use for a table.
* specified in SqoopOptions to determine the actual package and class name to
* use for a table.
*/
public class TableClassName {
public static final Log LOG = LogFactory.getLog(TableClassName.class.getName());
public static final Log LOG = LogFactory.getLog(
TableClassName.class.getName());
private final SqoopOptions options;
public TableClassName(final SqoopOptions opts) {
if (null == opts) {
throw new NullPointerException("Cannot instantiate a TableClassName on null options.");
throw new NullPointerException(
"Cannot instantiate a TableClassName on null options.");
} else {
this.options = opts;
}
@ -47,18 +49,19 @@ public TableClassName(final SqoopOptions opts) {
* package-part which will be used for a class. The actual table name being
* generated-for is irrelevant; so not an argument.
*
* @return the package where generated ORM classes go. Will be null for top-level.
* @return the package where generated ORM classes go. Will be null for
* top-level.
*/
public String getPackageForTable() {
String predefinedClass = options.getClassName();
if (null != predefinedClass) {
// if the predefined classname contains a package-part, return that.
// If the predefined classname contains a package-part, return that.
int lastDot = predefinedClass.lastIndexOf('.');
if (-1 == lastDot) {
// no package part.
// No package part.
return null;
} else {
// return the string up to but not including the last dot.
// Return the string up to but not including the last dot.
return predefinedClass.substring(0, lastDot);
}
} else {
@ -69,8 +72,8 @@ public String getPackageForTable() {
}
/**
* @param tableName the name of the table being imported
* @return the full name of the class to generate/use to import a table
* @param tableName the name of the table being imported.
* @return the full name of the class to generate/use to import a table.
*/
public String getClassForTable(String tableName) {
if (null == tableName) {
@ -95,7 +98,8 @@ public String getClassForTable(String tableName) {
}
/**
* @return just the last spegment of the class name -- all package info stripped.
* @return just the last spegment of the class name -- all package info
* stripped.
*/
public String getShortClassForTable(String tableName) {
String fullClass = getClassForTable(tableName);

5
src/java/org/apache/hadoop/sqoop/shims/HadoopShim.java
View File

@ -77,7 +77,8 @@ public abstract long getNumMapInputRecords(Job job)
/**
* Set the mapper speculative execution property for a job.
*/
public abstract void setJobMapSpeculativeExecution(Job job, boolean isEnabled);
public abstract void setJobMapSpeculativeExecution(Job job,
boolean isEnabled);
/**
* Sets the Jobtracker address to use for a job.
@ -123,7 +124,7 @@ public abstract long getNumMapInputRecords(Job job)
public abstract MapContext getMapContextForIOPath(
Configuration conf, Path p);
public final static synchronized HadoopShim get() {
public static final synchronized HadoopShim get() {
return ShimLoader.getHadoopShim(null);
}
}

4
src/java/org/apache/hadoop/sqoop/shims/ShimLoader.java
View File

@ -174,8 +174,8 @@ private static <T> T loadShim(List<String> matchExprs,
return shim;
} catch (Exception e) {
throw new RuntimeException("Could not load shim in class " +
className, e);
throw new RuntimeException("Could not load shim in class "
+ className, e);
}
}
}

14
src/java/org/apache/hadoop/sqoop/tool/BaseSqoopTool.java
View File

@ -52,7 +52,8 @@
*/
public abstract class BaseSqoopTool extends SqoopTool {
public static final Log LOG = LogFactory.getLog(BaseSqoopTool.class.getName());
public static final Log LOG = LogFactory.getLog(
BaseSqoopTool.class.getName());
public static final String HELP_STR = "\nTry --help for usage instructions.";
@ -248,7 +249,8 @@ protected RelatedOptions getCommonOptions() {
.hasArg().withDescription("Set authentication password")
.withLongOpt(PASSWORD_ARG)
.create());
commonOpts.addOption(OptionBuilder.withDescription("Read password from console")
commonOpts.addOption(OptionBuilder
.withDescription("Read password from console")
.create(PASSWORD_PROMPT_ARG));
commonOpts.addOption(OptionBuilder.withArgName("dir")
@ -342,10 +344,11 @@ protected RelatedOptions getOutputFormatOptions() {
}
/**
* @return options governing input format delimiters
* @return options governing input format delimiters.
*/
protected RelatedOptions getInputFormatOptions() {
RelatedOptions inputFormatOpts = new RelatedOptions("Input parsing arguments");
RelatedOptions inputFormatOpts =
new RelatedOptions("Input parsing arguments");
inputFormatOpts.addOption(OptionBuilder.withArgName("char")
.hasArg()
.withDescription("Sets the input field separator")
@ -380,7 +383,8 @@ protected RelatedOptions getInputFormatOptions() {
* @return options related to code generation.
*/
protected RelatedOptions getCodeGenOpts(boolean multiTable) {
RelatedOptions codeGenOpts = new RelatedOptions("Code generation arguments");
RelatedOptions codeGenOpts =
new RelatedOptions("Code generation arguments");
codeGenOpts.addOption(OptionBuilder.withArgName("dir")
.hasArg()
.withDescription("Output directory for generated code")

2
src/java/org/apache/hadoop/sqoop/tool/CodeGenTool.java
View File

@ -62,7 +62,7 @@ public List<String> getGeneratedJarFiles() {
}
/**
* Generate the .class and .jar files
* Generate the .class and .jar files.
* @return the filename of the emitted jar file.
* @throws IOException
*/

7
src/java/org/apache/hadoop/sqoop/tool/ExportTool.java
View File

@ -45,7 +45,6 @@
import org.apache.hadoop.sqoop.orm.ClassWriter;
import org.apache.hadoop.sqoop.orm.CompilationManager;
import org.apache.hadoop.sqoop.shims.ShimLoader;
import org.apache.hadoop.sqoop.tool.SqoopTool;
import org.apache.hadoop.sqoop.util.ExportException;
import org.apache.hadoop.sqoop.util.ImportException;
@ -77,7 +76,8 @@ private void exportTable(SqoopOptions options, String tableName)
// Generate the ORM code for the tables.
jarFile = codeGenerator.generateORM(options, tableName);
ExportJobContext context = new ExportJobContext(tableName, jarFile, options);
ExportJobContext context = new ExportJobContext(tableName, jarFile,
options);
manager.exportTable(context);
}
@ -94,7 +94,8 @@ public int run(SqoopOptions options) {
try {
exportTable(options, options.getTableName());
} catch (IOException ioe) {
LOG.error("Encountered IOException running export job: " + ioe.toString());
LOG.error("Encountered IOException running export job: "
+ ioe.toString());
if (System.getProperty(Sqoop.SQOOP_RETHROW_PROPERTY) != null) {
throw new RuntimeException(ioe);
} else {

1
src/java/org/apache/hadoop/sqoop/tool/HelpTool.java
View File

@ -21,7 +21,6 @@
import java.util.Set;
import org.apache.hadoop.sqoop.SqoopOptions;
import org.apache.hadoop.sqoop.cli.RelatedOptions;
import org.apache.hadoop.sqoop.cli.ToolOptions;
/**

22
src/java/org/apache/hadoop/sqoop/tool/ImportAllTablesTool.java
View File

@ -19,32 +19,13 @@
package org.apache.hadoop.sqoop.tool;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.sqoop.Sqoop;
import org.apache.hadoop.sqoop.SqoopOptions;
import org.apache.hadoop.sqoop.SqoopOptions.InvalidOptionsException;
import org.apache.hadoop.sqoop.hive.HiveImport;
import org.apache.hadoop.sqoop.manager.ConnManager;
import org.apache.hadoop.sqoop.manager.ExportJobContext;
import org.apache.hadoop.sqoop.manager.ImportJobContext;
import org.apache.hadoop.sqoop.orm.ClassWriter;
import org.apache.hadoop.sqoop.orm.CompilationManager;
import org.apache.hadoop.sqoop.shims.ShimLoader;
import org.apache.hadoop.sqoop.tool.SqoopTool;
import org.apache.hadoop.sqoop.util.ExportException;
import org.apache.hadoop.sqoop.util.ImportException;
/**
@ -84,7 +65,8 @@ public int run(SqoopOptions options) {
}
}
} catch (IOException ioe) {
LOG.error("Encountered IOException running import job: " + ioe.toString());
LOG.error("Encountered IOException running import job: "
+ ioe.toString());
if (System.getProperty(Sqoop.SQOOP_RETHROW_PROPERTY) != null) {
throw new RuntimeException(ioe);
} else {

6
src/java/org/apache/hadoop/sqoop/tool/ImportTool.java
View File

@ -84,7 +84,8 @@ protected void importTable(SqoopOptions options, String tableName,
jarFile = codeGenerator.generateORM(options, tableName);
// Do the actual import.
ImportJobContext context = new ImportJobContext(tableName, jarFile, options);
ImportJobContext context = new ImportJobContext(tableName, jarFile,
options);
manager.importTable(context);
// If the user wants this table to be in Hive, perform that post-load.
@ -120,7 +121,8 @@ public int run(SqoopOptions options) {
// Import a single table the user specified.
importTable(options, options.getTableName(), hiveImport);
} catch (IOException ioe) {
LOG.error("Encountered IOException running import job: " + ioe.toString());
LOG.error("Encountered IOException running import job: "
+ ioe.toString());
if (System.getProperty(Sqoop.SQOOP_RETHROW_PROPERTY) != null) {
throw new RuntimeException(ioe);
} else {

2
src/java/org/apache/hadoop/sqoop/tool/ListDatabasesTool.java
View File

@ -28,7 +28,7 @@
import org.apache.hadoop.sqoop.cli.ToolOptions;
/**
* Tool that lists available databases on a server
* Tool that lists available databases on a server.
*/
public class ListDatabasesTool extends BaseSqoopTool {

5
src/java/org/apache/hadoop/sqoop/tool/SqoopTool.java
View File

@ -83,7 +83,7 @@ public abstract class SqoopTool {
* @param cls the class providing the tool.
* @param description a user-friendly description of the tool's function.
*/
private static final void registerTool(String toolName,
private static void registerTool(String toolName,
Class<? extends SqoopTool> cls, String description) {
TOOLS.put(toolName, cls);
DESCRIPTIONS.put(toolName, description);
@ -169,7 +169,8 @@ public void configureOptions(ToolOptions opts) {
* @param opts the configured tool options
*/
public void printHelp(ToolOptions opts) {
System.out.println("usage: sqoop " + getToolName() + " [GENERIC-ARGS] [TOOL-ARGS]");
System.out.println("usage: sqoop " + getToolName()
+ " [GENERIC-ARGS] [TOOL-ARGS]");
System.out.println("");
opts.printHelp();

3
src/java/org/apache/hadoop/sqoop/util/AsyncSink.java
View File

@ -37,7 +37,8 @@ public abstract class AsyncSink {
/**
* Wait until the stream has been processed.
* @return a status code indicating success or failure. 0 is typical for success.
* @return a status code indicating success or failure. 0 is typical for
* success.
*/
public abstract int join() throws InterruptedException;
}

31
src/java/org/apache/hadoop/sqoop/util/ClassLoaderStack.java
View File

@ -29,19 +29,17 @@
/**
* Allows you to add and remove jar-files from the running JVM by
* instantiating classloaders for them.
*
*
*
*/
public final class ClassLoaderStack {
public static final Log LOG = LogFactory.getLog(ClassLoaderStack.class.getName());
public static final Log LOG = LogFactory.getLog(
ClassLoaderStack.class.getName());
private ClassLoaderStack() {
}
/**
* Sets the classloader for the current thread
* Sets the classloader for the current thread.
*/
public static void setCurrentClassLoader(ClassLoader cl) {
LOG.debug("Restoring classloader: " + cl.toString());
@ -49,23 +47,27 @@ public static void setCurrentClassLoader(ClassLoader cl) {
}
/**
* Adds a ClassLoader to the top of the stack that will load from the Jar file
* of your choice. Returns the previous classloader so you can restore it
* if need be, later.
* Adds a ClassLoader to the top of the stack that will load from the Jar
* file of your choice. Returns the previous classloader so you can restore
* it if need be, later.
*
* @param jarFile The filename of a jar file that you want loaded into this JVM
* @param testClassName The name of the class to load immediately (optional)
* @param jarFile The filename of a jar file that you want loaded into this
* JVM.
* @param testClassName The name of the class to load immediately
* (optional).
*/
public static ClassLoader addJarFile(String jarFile, String testClassName)
throws IOException {
// load the classes from the ORM JAR file into the current VM
ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader();
// load the classes from the ORM JAR file into the current VM.
ClassLoader prevClassLoader =
Thread.currentThread().getContextClassLoader();
String urlPath = "jar:file://" + new File(jarFile).getAbsolutePath() + "!/";
LOG.debug("Attempting to load jar through URL: " + urlPath);
LOG.debug("Previous classloader is " + prevClassLoader);
URL [] jarUrlArray = {new URL(urlPath)};
URLClassLoader cl = URLClassLoader.newInstance(jarUrlArray, prevClassLoader);
URLClassLoader cl = URLClassLoader.newInstance(jarUrlArray,
prevClassLoader);
try {
if (null != testClassName) {
// try to load a class from the jar to force loading now.
@ -74,7 +76,8 @@ public static ClassLoader addJarFile(String jarFile, String testClassName)
}
LOG.debug("Loaded jar into current JVM: " + urlPath);
} catch (ClassNotFoundException cnfe) {
throw new IOException("Could not load jar " + jarFile + " into JVM. (Could not find class "
throw new IOException("Could not load jar " + jarFile
+ " into JVM. (Could not find class "
+ testClassName + ".)", cnfe);
}

2
src/java/org/apache/hadoop/sqoop/util/ErrorableAsyncSink.java
View File

@ -18,8 +18,6 @@
package org.apache.hadoop.sqoop.util;
import java.io.InputStream;
/**
* Partial implementation of AsyncSink that relies on ErrorableThread to
* provide a status bit for the join() method.

18
src/java/org/apache/hadoop/sqoop/util/FileListing.java
View File

@ -61,15 +61,16 @@ public static void main(String... aArgs) throws FileNotFoundException {
*
* @param aStartingDir is a valid directory, which can be read.
*/
public static List<File> getFileListing(File aStartingDir) throws FileNotFoundException {
public static List<File> getFileListing(File aStartingDir)
throws FileNotFoundException {
validateDirectory(aStartingDir);
List<File> result = getFileListingNoSort(aStartingDir);
Collections.sort(result);
return result;
}
// PRIVATE //
private static List<File> getFileListingNoSort(File aStartingDir) throws FileNotFoundException {
private static List<File> getFileListingNoSort(File aStartingDir)
throws FileNotFoundException {
List<File> result = new ArrayList<File>();
File[] filesAndDirs = aStartingDir.listFiles();
List<File> filesDirs = Arrays.asList(filesAndDirs);
@ -88,23 +89,26 @@ private static List<File> getFileListingNoSort(File aStartingDir) throws FileNot
/**
* Directory is valid if it exists, does not represent a file, and can be read.
*/
private static void validateDirectory(File aDirectory) throws FileNotFoundException {
private static void validateDirectory(File aDirectory)
throws FileNotFoundException {
if (aDirectory == null) {
throw new IllegalArgumentException("Directory should not be null.");
}
if (!aDirectory.exists()) {
throw new FileNotFoundException("Directory does not exist: " + aDirectory);
throw new FileNotFoundException("Directory does not exist: "
+ aDirectory);
}
if (!aDirectory.isDirectory()) {
throw new IllegalArgumentException("Is not a directory: " + aDirectory);
}
if (!aDirectory.canRead()) {
throw new IllegalArgumentException("Directory cannot be read: " + aDirectory);
throw new IllegalArgumentException("Directory cannot be read: "
+ aDirectory);
}
}
/**
* Recursively delete a directory and all its children
* Recursively delete a directory and all its children.
* @param dir is a valid directory.
*/
public static void recursiveDeleteDir(File dir) throws IOException {

13
src/java/org/apache/hadoop/sqoop/util/JdbcUrl.java
View File

@ -38,8 +38,8 @@ private JdbcUrl() {
}
/**
* @return the database name from the connect string, which is typically the 'path'
* component, or null if we can't.
* @return the database name from the connect string, which is typically the
* 'path' component, or null if we can't.
*/
public static String getDatabaseName(String connectString) {
try {
@ -48,7 +48,8 @@ public static String getDatabaseName(String connectString) {
if (-1 == schemeEndOffset) {
// couldn't find one? try our best here.
sanitizedString = "http://" + connectString;
LOG.warn("Could not find database access scheme in connect string " + connectString);
LOG.warn("Could not find database access scheme in connect string "
+ connectString);
} else {
sanitizedString = "http" + connectString.substring(schemeEndOffset);
}
@ -81,7 +82,8 @@ public static String getHostName(String connectString) {
String sanitizedString = null;
int schemeEndOffset = connectString.indexOf("://");
if (-1 == schemeEndOffset) {
// couldn't find one? ok, then there's no problem, it should work as a URL.
// Couldn't find one? ok, then there's no problem, it should work as a
// URL.
sanitizedString = connectString;
} else {
sanitizedString = "http" + connectString.substring(schemeEndOffset);
@ -104,7 +106,8 @@ public static int getPort(String connectString) {
String sanitizedString = null;
int schemeEndOffset = connectString.indexOf("://");
if (-1 == schemeEndOffset) {
// couldn't find one? ok, then there's no problem, it should work as a URL.
// Couldn't find one? ok, then there's no problem, it should work as a
// URL.
sanitizedString = connectString;
} else {
sanitizedString = "http" + connectString.substring(schemeEndOffset);

3
src/java/org/apache/hadoop/sqoop/util/LoggingAsyncSink.java
View File

@ -32,7 +32,8 @@
*/
public class LoggingAsyncSink extends AsyncSink {
public static final Log LOG = LogFactory.getLog(LoggingAsyncSink.class.getName());
public static final Log LOG = LogFactory.getLog(
LoggingAsyncSink.class.getName());
private Log contextLog;

6
src/java/org/apache/hadoop/sqoop/util/NullAsyncSink.java
View File

@ -31,7 +31,8 @@
*/
public class NullAsyncSink extends AsyncSink {
public static final Log LOG = LogFactory.getLog(NullAsyncSink.class.getName());
public static final Log LOG = LogFactory.getLog(
NullAsyncSink.class.getName());
private Thread child;
@ -69,7 +70,8 @@ public void run() {
}
}
} catch (IOException ioe) {
LOG.warn("IOException reading from (ignored) stream: " + ioe.toString());
LOG.warn("IOException reading from (ignored) stream: "
+ ioe.toString());
}
try {

9
src/java/org/apache/hadoop/sqoop/util/PerfCounters.java
View File

@ -47,7 +47,7 @@ public void stopClock() {
private static final double ONE_BILLION = 1000.0 * 1000.0 * 1000.0;
/** maximum number of digits after the decimal place */
/** Maximum number of digits after the decimal place. */
private static final int MAX_PLACES = 4;
/**
@ -63,8 +63,8 @@ private Double inSeconds(long nanos) {
/**
* @return a string of the form "xxxx bytes" or "xxxxx KB" or "xxxx GB", scaled
* as is appropriate for the current value.
* @return a string of the form "xxxx bytes" or "xxxxx KB" or "xxxx GB",
* scaled as is appropriate for the current value.
*/
private String formatBytes() {
double val;
@ -125,7 +125,8 @@ private String formatSpeed() {
}
public String toString() {
return formatBytes() + " in " + formatTimeInSeconds() + " (" + formatSpeed() + ")";
return formatBytes() + " in " + formatTimeInSeconds() + " ("
+ formatSpeed() + ")";
}
}

10
src/java/org/apache/hadoop/sqoop/util/ResultSetPrinter.java
View File

@ -29,17 +29,18 @@
import org.apache.hadoop.util.StringUtils;
/**
* Utility methods to format and print ResultSet objects
* Utility methods to format and print ResultSet objects.
*/
public class ResultSetPrinter {
public static final Log LOG = LogFactory.getLog(ResultSetPrinter.class.getName());
public static final Log LOG = LogFactory.getLog(
ResultSetPrinter.class.getName());
// max output width to allocate to any column of the printed results.
private static final int MAX_COL_WIDTH = 20;
/**
* Print 'str' to the string builder, padded to 'width' chars
* Print 'str' to the string builder, padded to 'width' chars.
*/
private static void printPadded(StringBuilder sb, String str, int width) {
int numPad;
@ -72,7 +73,8 @@ public final void printResultSet(PrintWriter pw, ResultSet results)
ResultSetMetaData metadata = results.getMetaData();
for (int i = 1; i < cols + 1; i++) {
String colName = metadata.getColumnName(i);
colWidths[i - 1] = Math.min(metadata.getColumnDisplaySize(i), MAX_COL_WIDTH);
colWidths[i - 1] = Math.min(metadata.getColumnDisplaySize(i),
MAX_COL_WIDTH);
if (colName == null || colName.equals("")) {
colName = metadata.getColumnLabel(i) + "*";
}

22
src/perftest/ExportStressTest.java
View File

@ -30,19 +30,20 @@
/**
* Stress test export procedure by running a large-scale export to MySQL.
* This requires MySQL be configured with a database that can be accessed
* by the specified username without a password. The user must be able to
* create and drop tables in the database.
* This requires MySQL be configured with a database that can be accessed by
* the specified username without a password. The user must be able to create
* and drop tables in the database.
*
* Run with: src/scripts/run-perftest.sh ExportStressTest (connect-str) (username)
* Run with: src/scripts/run-perftest.sh ExportStressTest \
* (connect-str) (username)
*/
public class ExportStressTest extends Configured implements Tool {
// Export 10 GB of data. Each record is ~100 bytes.
public final static int NUM_FILES = 10;
public final static int RECORDS_PER_FILE = 10 * 1024 * 1024;
public static final int NUM_FILES = 10;
public static final int RECORDS_PER_FILE = 10 * 1024 * 1024;
public final static String ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
public static final String ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
public ExportStressTest() {
}
@ -88,7 +89,8 @@ public void createTable(String connectStr, String username) throws Exception {
Connection conn = DriverManager.getConnection(connectStr, username, null);
conn.setAutoCommit(false);
PreparedStatement stmt = conn.prepareStatement(
"DROP TABLE IF EXISTS ExportStressTestTable", ResultSet.TYPE_FORWARD_ONLY,
"DROP TABLE IF EXISTS ExportStressTestTable",
ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY);
stmt.executeUpdate();
stmt.close();
@ -103,7 +105,9 @@ public void createTable(String connectStr, String username) throws Exception {
conn.close();
}
/** Actually run the export of the generated data to the user-created table. */
/**
* Actually run the export of the generated data to the user-created table.
*/
public void runExport(String connectStr, String username) throws Exception {
SqoopOptions options = new SqoopOptions(getConf());
options.setConnectString(connectStr);

17
src/perftest/LobFilePerfTest.java
View File

@ -25,16 +25,17 @@
* A simple benchmark to performance test LobFile reader/writer speed.
* Writes out 10 GB of data to the local disk and then reads it back.
* Run with:
* HADOOP_OPTS=-agentlib:hprof=cpu=samples src/scripts/run-perftest.sh LobFilePerfTest
* HADOOP_OPTS=-agentlib:hprof=cpu=samples \
* src/scripts/run-perftest.sh LobFilePerfTest
*/
public class LobFilePerfTest {
long recordLen = 20 * 1024 * 1024; // 20 MB records
int numRecords = 500;
Configuration conf;
Path p;
long startTime;
byte [] record;
private long recordLen = 20 * 1024 * 1024; // 20 MB records
private int numRecords = 500;
private Configuration conf;
private Path p;
private long startTime;
private byte [] record;
public LobFilePerfTest() {
conf = new Configuration();
@ -99,7 +100,7 @@ private void readFile() throws Exception {
System.out.println("Read " + recordSize + " bytes");
}
private void run() throws Exception {
public void run() throws Exception {
makeRecordBody();
writeFile();
readFile();

17
src/perftest/LobFileStressTest.java
View File

@ -30,14 +30,14 @@
public class LobFileStressTest {
// Big records in testBigFile() are 5 GB each.
public final static long LARGE_RECORD_LEN = 5L * 1024L * 1024L * 1024L;
public static final long LARGE_RECORD_LEN = 5L * 1024L * 1024L * 1024L;
int numRandomTrials = 1000000;
Configuration conf;
boolean allPassed;
private int numRandomTrials = 1000000;
private Configuration conf;
private boolean allPassed;
long lastCompressPos; // start offset of the last record in the file.
long lastRawPos;
private long lastCompressPos; // start offset of the last record in the file.
private long lastRawPos;
public LobFileStressTest() {
conf = new Configuration();
@ -314,7 +314,7 @@ private void testBigFile(boolean compress) throws Exception {
+ compress + ". ");
Path p = getBigFilePath(compress);
long startOffsets [] = new long[NUM_RECORDS];
long [] startOffsets = new long[NUM_RECORDS];
// Write the file. Five records, 5 GB a piece.
System.out.print("Testing write. ");
@ -364,8 +364,7 @@ private void testBigFile(boolean compress) throws Exception {
}
}
private void run() throws Exception {
public void run() throws Exception {
writeIntegerFile(true);
writeIntegerFile(false);
testSequentialScan(false);

2
src/shims/cloudera/org/apache/hadoop/sqoop/shims/CDH3Shim.java
View File

@ -33,7 +33,7 @@
/**
* Hadoop Shim for CDH3 (based on 0.20.2)
* Hadoop Shim for CDH3 (based on 0.20.2).
*/
public class CDH3Shim extends CommonHadoopShim {
@Override

9
src/shims/common/org/apache/hadoop/sqoop/mapreduce/CombineShimRecordReader.java
View File

@ -24,7 +24,6 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
@ -65,10 +64,10 @@ public CombineShimRecordReader(CombineFileSplit split,
}
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
public void initialize(InputSplit curSplit, TaskAttemptContext curContext)
throws IOException, InterruptedException {
this.split = (CombineFileSplit) split;
this.context = context;
this.split = (CombineFileSplit) curSplit;
this.context = curContext;
if (null == rr) {
createChildReader();
@ -77,7 +76,7 @@ public void initialize(InputSplit split, TaskAttemptContext context)
FileSplit fileSplit = new FileSplit(this.split.getPath(index),
this.split.getOffset(index), this.split.getLength(index),
this.split.getLocations());
this.rr.initialize(fileSplit, context);
this.rr.initialize(fileSplit, this.context);
}
@Override

5
src/shims/common/org/apache/hadoop/sqoop/mapreduce/ExportInputFormat.java
View File

@ -18,18 +18,13 @@
package org.apache.hadoop.sqoop.mapreduce;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;

12
src/shims/common/org/apache/hadoop/sqoop/mapreduce/ExportOutputFormat.java
View File

@ -1,8 +1,8 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* Licensed to Cloudera, Inc. under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* regarding copyright ownership. Cloudera, Inc. licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
@ -29,7 +29,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.OutputFormat;
@ -52,7 +51,7 @@
* Uses DBOutputFormat/DBConfiguration for configuring the output.
*/
public class ExportOutputFormat<K extends SqoopRecord, V>
extends OutputFormat<K,V> {
extends OutputFormat<K, V> {
/** conf key: number of rows to export per INSERT statement. */
public static final String RECORDS_PER_STATEMENT_KEY =
@ -72,6 +71,7 @@ public class ExportOutputFormat<K extends SqoopRecord, V>
private static final Log LOG = LogFactory.getLog(ExportOutputFormat.class);
@Override
/** {@inheritDoc} */
public void checkOutputSpecs(JobContext context)
throws IOException, InterruptedException {
@ -90,6 +90,7 @@ public void checkOutputSpecs(JobContext context)
}
}
@Override
/** {@inheritDoc} */
public OutputCommitter getOutputCommitter(TaskAttemptContext context)
throws IOException, InterruptedException {
@ -105,6 +106,7 @@ public void setupTask(TaskAttemptContext taskContext) { }
};
}
@Override
/** {@inheritDoc} */
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context)
throws IOException {
@ -454,6 +456,7 @@ private void insertRows(boolean closeConn)
}
}
@Override
/** {@inheritDoc} */
public void close(TaskAttemptContext context)
throws IOException, InterruptedException {
@ -474,6 +477,7 @@ public void close(TaskAttemptContext context)
}
}
@Override
/** {@inheritDoc} */
public void write(K key, V value)
throws InterruptedException, IOException {

26
src/shims/common/org/apache/hadoop/sqoop/mapreduce/MySQLDumpInputFormat.java
View File

@ -18,40 +18,14 @@
package org.apache.hadoop.sqoop.mapreduce;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.CharBuffer;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat;
import org.apache.hadoop.sqoop.SqoopOptions;
import org.apache.hadoop.sqoop.io.SplittableBufferedWriter;
import org.apache.hadoop.sqoop.lib.FieldFormatter;
import org.apache.hadoop.sqoop.lib.RecordParser;
import org.apache.hadoop.sqoop.util.AsyncSink;
import org.apache.hadoop.sqoop.util.DirectImportUtils;
import org.apache.hadoop.sqoop.util.ErrorableAsyncSink;
import org.apache.hadoop.sqoop.util.ErrorableThread;
import org.apache.hadoop.sqoop.util.ImportException;
import org.apache.hadoop.sqoop.util.JdbcUrl;
import org.apache.hadoop.sqoop.util.LoggingAsyncSink;
import org.apache.hadoop.sqoop.util.PerfCounters;
/**
* InputFormat designed to take data-driven splits and feed them to a mysqldump

7
src/shims/common/org/apache/hadoop/sqoop/mapreduce/OracleExportOutputFormat.java
View File

@ -1,8 +1,8 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* Licensed to Cloudera, Inc. under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* regarding copyright ownership. Cloudera, Inc. licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
@ -30,8 +30,9 @@
* Oracle-specific SQL formatting overrides default ExportOutputFormat's.
*/
public class OracleExportOutputFormat<K extends SqoopRecord, V>
extends ExportOutputFormat<K,V> {
extends ExportOutputFormat<K, V> {
@Override
/** {@inheritDoc} */
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context)
throws IOException {

8
src/shims/common/org/apache/hadoop/sqoop/mapreduce/RawKeyTextOutputFormat.java
View File

@ -28,7 +28,6 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
@ -40,7 +39,7 @@
public class RawKeyTextOutputFormat<K, V> extends FileOutputFormat<K, V> {
protected static class RawKeyRecordWriter<K, V> extends RecordWriter<K, V> {
private static final String utf8 = "UTF-8";
private static final String UTF8 = "UTF-8";
protected DataOutputStream out;
@ -59,7 +58,7 @@ private void writeObject(Object o) throws IOException {
Text to = (Text) o;
out.write(to.getBytes(), 0, to.getLength());
} else {
out.write(o.toString().getBytes(utf8));
out.write(o.toString().getBytes(UTF8));
}
}
@ -67,7 +66,8 @@ public synchronized void write(K key, V value) throws IOException {
writeObject(key);
}
public synchronized void close(TaskAttemptContext context) throws IOException {
public synchronized void close(TaskAttemptContext context)
throws IOException {
out.close();
}
}

17
src/test/checkstyle-java-header.txt Normal file
View File

@ -0,0 +1,17 @@
/**
* Licensed to Cloudera, Inc. under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Cloudera, Inc. licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

196
src/test/checkstyle-noframes.xsl Normal file
View File

@ -0,0 +1,196 @@
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
<!--
Licensed to Cloudera, Inc. under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<xsl:output method="html" indent="yes"/>
<xsl:decimal-format decimal-separator="." grouping-separator="," />
<xsl:key name="files" match="file" use="@name" />
<!-- Checkstyle XML Style Sheet by Stephane Bailliez <sbailliez@apache.org> -->
<!-- Part of the Checkstyle distribution found at http://checkstyle.sourceforge.net -->
<!-- Usage (generates checkstyle_report.html): -->
<!-- <checkstyle failonviolation="false" config="${check.config}"> -->
<!-- <fileset dir="${src.dir}" includes="**/*.java"/> -->
<!-- <formatter type="xml" toFile="${doc.dir}/checkstyle_report.xml"/> -->
<!-- </checkstyle> -->
<!-- <style basedir="${doc.dir}" destdir="${doc.dir}" -->
<!-- includes="checkstyle_report.xml" -->
<!-- style="${doc.dir}/checkstyle-noframes-sorted.xsl"/> -->
<xsl:template match="checkstyle">
<html>
<head>
<style type="text/css">
.bannercell {
border: 0px;
padding: 0px;
}
body {
margin-left: 10;
margin-right: 10;
font:normal 80% arial,helvetica,sanserif;
background-color:#FFFFFF;
color:#000000;
}
.a td {
background: #efefef;
}
.b td {
background: #fff;
}
th, td {
text-align: left;
vertical-align: top;
}
th {
font-weight:bold;
background: #ccc;
color: black;
}
table, th, td {
font-size:100%;
border: none
}
table.log tr td, tr th {
}
h2 {
font-weight:bold;
font-size:140%;
margin-bottom: 5;
}
h3 {
font-size:100%;
font-weight:bold;
background: #525D76;
color: white;
text-decoration: none;
padding: 5px;
margin-right: 2px;
margin-left: 2px;
margin-bottom: 0;
}
</style>
</head>
<body>
<a name="top"></a>
<!-- jakarta logo -->
<table border="0" cellpadding="0" cellspacing="0" width="100%">
<tr>
<td class="bannercell" rowspan="2">
<!--a href="http://jakarta.apache.org/">
<img src="http://jakarta.apache.org/images/jakarta-logo.gif" alt="http://jakarta.apache.org" align="left" border="0"/>
</a-->
</td>
<td class="text-align:right"><h2>CheckStyle Audit</h2></td>
</tr>
<tr>
<td class="text-align:right">Designed for use with <a href='http://checkstyle.sourceforge.net/'>CheckStyle</a> and <a href='http://jakarta.apache.org'>Ant</a>.</td>
</tr>
</table>
<hr size="1"/>
<!-- Summary part -->
<xsl:apply-templates select="." mode="summary"/>
<hr size="1" width="100%" align="left"/>
<!-- Package List part -->
<xsl:apply-templates select="." mode="filelist"/>
<hr size="1" width="100%" align="left"/>
<!-- For each package create its part -->
<xsl:apply-templates select="file[@name and generate-id(.) = generate-id(key('files', @name))]" />
<hr size="1" width="100%" align="left"/>
</body>
</html>
</xsl:template>
<xsl:template match="checkstyle" mode="filelist">
<h3>Files</h3>
<table class="log" border="0" cellpadding="5" cellspacing="2" width="100%">
<tr>
<th>Name</th>
<th>Errors</th>
</tr>
<xsl:for-each select="file[@name and generate-id(.) = generate-id(key('files', @name))]">
<xsl:sort data-type="number" order="descending" select="count(key('files', @name)/error)"/>
<xsl:variable name="errorCount" select="count(error)"/>
<tr>
<xsl:call-template name="alternated-row"/>
<td><a href="#f-{@name}"><xsl:value-of select="@name"/></a></td>
<td><xsl:value-of select="$errorCount"/></td>
</tr>
</xsl:for-each>
</table>
</xsl:template>
<xsl:template match="file">
<a name="f-{@name}"></a>
<h3>File <xsl:value-of select="@name"/></h3>
<table class="log" border="0" cellpadding="5" cellspacing="2" width="100%">
<tr>
<th>Error Description</th>
<th>Line</th>
</tr>
<xsl:for-each select="key('files', @name)/error">
<xsl:sort data-type="number" order="ascending" select="@line"/>
<tr>
<xsl:call-template name="alternated-row"/>
<td><xsl:value-of select="@message"/></td>
<td><xsl:value-of select="@line"/></td>
</tr>
</xsl:for-each>
</table>
<a href="#top">Back to top</a>
</xsl:template>
<xsl:template match="checkstyle" mode="summary">
<h3>Summary</h3>
<xsl:variable name="fileCount" select="count(file[@name and generate-id(.) = generate-id(key('files', @name))])"/>
<xsl:variable name="errorCount" select="count(file/error)"/>
<table class="log" border="0" cellpadding="5" cellspacing="2" width="100%">
<tr>
<th>Files</th>
<th>Errors</th>
</tr>
<tr>
<xsl:call-template name="alternated-row"/>
<td><xsl:value-of select="$fileCount"/></td>
<td><xsl:value-of select="$errorCount"/></td>
</tr>
</table>
</xsl:template>
<xsl:template name="alternated-row">
<xsl:attribute name="class">
<xsl:if test="position() mod 2 = 1">a</xsl:if>
<xsl:if test="position() mod 2 = 0">b</xsl:if>
</xsl:attribute>
</xsl:template>
</xsl:stylesheet>

209
src/test/checkstyle.xml Normal file
View File

@ -0,0 +1,209 @@
<?xml version="1.0"?>
<!--
Licensed to Cloudera, Inc. under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.2//EN"
"http://www.puppycrawl.com/dtds/configuration_1_2.dtd">
<!--
Checkstyle configuration for Sqoop that is based on the sun_checks.xml file
that is bundled with Checkstyle and includes checks for:
- the Java Language Specification at
http://java.sun.com/docs/books/jls/second_edition/html/index.html
- the Sun Code Conventions at http://java.sun.com/docs/codeconv/
- the Javadoc guidelines at
http://java.sun.com/j2se/javadoc/writingdoccomments/index.html
- the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html
- some best practices
Checkstyle is very configurable. Be sure to read the documentation at
http://checkstyle.sf.net (or in your downloaded distribution).
Most Checks are configurable, be sure to consult the documentation.
To completely disable a check, just comment it out or delete it from the file.
Finally, it is worth reading the documentation.
-->
<module name="Checker">
<!-- Checks that a package.html file exists for each package. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html#PackageHtml -->
<!-- module name="PackageHtml"/ -->
<!-- Checks whether files end with a new line. -->
<!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile -->
<module name="NewlineAtEndOfFile"/>
<!-- Checks for Headers -->
<!-- See http://checkstyle.sf.net/config_header.html -->
<module name="Header">
<property name="headerFile"
value="${test.dir}/checkstyle-java-header.txt" />
</module>
<module name="FileLength"/>
<module name="FileTabCharacter"/>
<module name="TreeWalker">
<!-- Checks for Javadoc comments. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html -->
<module name="JavadocType">
<property name="scope" value="public"/>
<property name="allowMissingParamTags" value="true"/>
</module>
<module name="JavadocStyle"/>
<module name="SuperClone"/>
<module name="SuperFinalize"/>
<!-- Checks for Naming Conventions. -->
<!-- See http://checkstyle.sf.net/config_naming.html -->
<module name="ConstantName"/>
<module name="ClassTypeParameterName">
<property name="format" value="^[A-Z]+$"/>
</module>
<module name="LocalFinalVariableName">
<property name="format" value="^[A-Z][_A-Z0-9]*$" />
</module>
<module name="LocalVariableName"/>
<module name="MemberName"/>
<module name="MethodName"/>
<module name="MethodTypeParameterName">
<property name="format" value="^[A-Z]+$"/>
</module>
<module name="PackageName"/>
<module name="ParameterName"/>
<module name="StaticVariableName"/>
<module name="TypeName"/>
<!-- Checks for imports -->
<!-- See http://checkstyle.sf.net/config_import.html -->
<module name="IllegalImport"/> <!-- defaults to sun.* packages -->
<module name="RedundantImport"/>
<module name="UnusedImports"/>
<!-- Checks for Size Violations. -->
<!-- See http://checkstyle.sf.net/config_sizes.html -->
<module name="LineLength"/>
<module name="MethodLength"/>
<module name="ParameterNumber"/>
<module name="OuterTypeNumber"/>
<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="GenericWhitespace"/>
<module name="EmptyForIteratorPad"/>
<module name="MethodParamPad"/>
<module name="NoWhitespaceAfter">
<property name="tokens"
value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS, UNARY_PLUS" />
</module>
<module name="NoWhitespaceBefore"/>
<module name="OperatorWrap"/>
<module name="ParenPad"/>
<module name="TypecastParenPad"/>
<module name="WhitespaceAfter">
<property name="tokens" value="COMMA, SEMI"/>
</module>
<!-- Modifier Checks -->
<!-- See http://checkstyle.sf.net/config_modifiers.html -->
<module name="ModifierOrder"/>
<module name="RedundantModifier"/>
<!-- Checks for blocks. You know, those {}'s -->
<!-- See http://checkstyle.sf.net/config_blocks.html -->
<module name="AvoidNestedBlocks"/>
<module name="EmptyBlock">
<!-- catch blocks need a statement or a comment. -->
<property name="option" value="text" />
<property name="tokens" value="LITERAL_CATCH" />
</module>
<module name="EmptyBlock">
<!-- all other blocks need a real statement. -->
<property name="option" value="stmt" />
<property name="tokens" value="LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY,
LITERAL_IF, LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT,
STATIC_INIT" />
</module>
<module name="LeftCurly"/>
<module name="NeedBraces"/>
<module name="RightCurly"/>
<!-- Checks for common coding problems -->
<!-- See http://checkstyle.sf.net/config_coding.html -->
<!-- module name="AvoidInlineConditionals"/-->
<module name="DoubleCheckedLocking"/>
<module name="EmptyStatement"/>
<module name="EqualsHashCode"/>
<module name="StringLiteralEquality" />
<module name="HiddenField">
<property name="ignoreConstructorParameter" value="true"/>
</module>
<module name="IllegalInstantiation"/>
<module name="InnerAssignment"/>
<module name="MissingSwitchDefault"/>
<module name="RedundantThrows"/>
<module name="SimplifyBooleanExpression"/>
<module name="SimplifyBooleanReturn"/>
<module name="DefaultComesLast" />
<!-- Checks for class design -->
<!-- See http://checkstyle.sf.net/config_design.html -->
<module name="FinalClass"/>
<module name="HideUtilityClassConstructor"/>
<module name="InterfaceIsType"/>
<module name="VisibilityModifier">
<property name="protectedAllowed" value="true" />
</module>
<module name="MissingOverride" />
<!-- Miscellaneous other checks. -->
<!-- See http://checkstyle.sf.net/config_misc.html -->
<module name="ArrayTypeStyle"/>
<module name="ArrayTrailingComma"/>
<!--
This generates too many false-positives on wrapped 'throws' clauses
to be really useful. Disabled for now.
Sqoop style is:
* Spaces, not tabs.
* Indent by two spaces.
* Indent by four spaces when wrapping a line.
<module name="Indentation">
<property name="basicOffset" value="2" />
<property name="caseIndent" value="0" />
</module>
-->
<!-- module name="TodoComment"/ -->
<module name="UpperEll"/>
</module>
</module>

2
src/test/org/apache/hadoop/sqoop/AllTests.java
View File

@ -22,7 +22,7 @@
import junit.framework.TestSuite;
/**
* All tests for Sqoop (org.apache.hadoop.sqoop)
* All tests for Sqoop (org.apache.hadoop.sqoop).
*/
public final class AllTests {

2
src/test/org/apache/hadoop/sqoop/SmokeTests.java
View File

@ -38,7 +38,7 @@
import junit.framework.TestSuite;
/**
* Smoke tests for Sqoop (org.apache.hadoop.sqoop)
* Smoke tests for Sqoop (org.apache.hadoop.sqoop).
*/
public final class SmokeTests {

7
src/test/org/apache/hadoop/sqoop/TestAllTables.java
View File

@ -42,7 +42,7 @@
public class TestAllTables extends ImportJobTestCase {
/**
* Create the argv to pass to Sqoop
* Create the argv to pass to Sqoop.
* @return the argv as an array of strings.
*/
private String [] getArgv(boolean includeHadoopFlags) {
@ -67,7 +67,7 @@ public class TestAllTables extends ImportJobTestCase {
/** the names of the tables we're creating. */
private List<String> tableNames;
/** The strings to inject in the (ordered) tables */
/** The strings to inject in the (ordered) tables. */
private List<String> expectedStrings;
@Before
@ -123,7 +123,8 @@ public void testMultiTableImport() throws IOException {
this.expectedStrings.remove(0);
BufferedReader reader = new BufferedReader(
new InputStreamReader(new FileInputStream(new File(filePath.toString()))));
new InputStreamReader(new FileInputStream(
new File(filePath.toString()))));
try {
String line = reader.readLine();
assertEquals("Table " + tableName + " expected a different string",

3
src/test/org/apache/hadoop/sqoop/TestColumnTypes.java
View File

@ -40,7 +40,8 @@
*/
public class TestColumnTypes extends ManagerCompatTestCase {
public static final Log LOG = LogFactory.getLog(TestColumnTypes.class.getName());
public static final Log LOG = LogFactory.getLog(
TestColumnTypes.class.getName());
@Override
protected Log getLogger() {

14
src/test/org/apache/hadoop/sqoop/TestConnFactory.java
View File

@ -38,7 +38,8 @@ public class TestConnFactory extends TestCase {
public void testCustomFactory() throws IOException {
Configuration conf = new Configuration();
conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, AlwaysDummyFactory.class.getName());
conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY,
AlwaysDummyFactory.class.getName());
ConnFactory factory = new ConnFactory(conf);
ConnManager manager = factory.getManager(new SqoopOptions());
@ -76,6 +77,10 @@ public void testMultipleManagers() throws IOException {
////// mock classes used for test cases above //////
/**
* Factory that always returns a DummyManager, regardless of the
* configuration.
*/
public static class AlwaysDummyFactory extends ManagerFactory {
public ConnManager accept(SqoopOptions opts) {
// Always return a new DummyManager
@ -83,6 +88,9 @@ public ConnManager accept(SqoopOptions opts) {
}
}
/**
* ManagerFactory that accepts no configurations.
*/
public static class EmptyFactory extends ManagerFactory {
public ConnManager accept(SqoopOptions opts) {
// Never instantiate a proper ConnManager;
@ -114,7 +122,7 @@ public String getPrimaryKey(String tableName) {
}
/**
* Default implementation
* Default implementation.
* @param sqlType sql data type
* @return java data type
*/
@ -123,7 +131,7 @@ public String toJavaType(int sqlType) {
}
/**
* Default implementation
* Default implementation.
* @param sqlType sql data type
* @return hive data type
*/

43
src/test/org/apache/hadoop/sqoop/TestExport.java
View File

@ -120,22 +120,22 @@ private String getRecordLine(int recordNum, ColumnGenerator... extraCols) {
the database should look like.
*/
public interface ColumnGenerator {
/** for a row with id rowNum, what should we write into that
/** For a row with id rowNum, what should we write into that
line of the text file to export?
*/
public String getExportText(int rowNum);
String getExportText(int rowNum);
/** for a row with id rowNum, what should the database return
/** For a row with id rowNum, what should the database return
for the given column's value?
*/
public String getVerifyText(int rowNum);
String getVerifyText(int rowNum);
/** Return the column type to put in the CREATE TABLE statement */
public String getType();
/** Return the column type to put in the CREATE TABLE statement. */
String getType();
}
/**
* Create a data file that gets exported to the db
* Create a data file that gets exported to the db.
* @param fileNum the number of the file (for multi-file export)
* @param numRecords how many records to write to the file.
* @param gzip is true if the file should be gzipped.
@ -173,7 +173,8 @@ private void createTextFile(int fileNum, int numRecords, boolean gzip,
}
}
private void verifyCompressedFile(Path f, int expectedNumLines) throws IOException {
private void verifyCompressedFile(Path f, int expectedNumLines)
throws IOException {
Configuration conf = new Configuration();
conf.set("fs.default.name", "file:///");
FileSystem fs = FileSystem.get(conf);
@ -185,7 +186,8 @@ private void verifyCompressedFile(Path f, int expectedNumLines) throws IOExcepti
if (null == decompressor) {
LOG.info("Verifying gzip sanity with null decompressor");
} else {
LOG.info("Verifying gzip sanity with decompressor: " + decompressor.toString());
LOG.info("Verifying gzip sanity with decompressor: "
+ decompressor.toString());
}
is = codec.createInputStream(is, decompressor);
BufferedReader r = new BufferedReader(new InputStreamReader(is));
@ -205,7 +207,7 @@ private void verifyCompressedFile(Path f, int expectedNumLines) throws IOExcepti
}
/**
* Create a data file in SequenceFile format that gets exported to the db
* Create a data file in SequenceFile format that gets exported to the db.
* @param fileNum the number of the file (for multi-file export).
* @param numRecords how many records to write to the file.
* @param className the table class name to instantiate and populate
@ -303,7 +305,7 @@ public void createTable(ColumnGenerator... extraColumns) throws SQLException {
}
}
/** Removing an existing table directory from the filesystem */
/** Removing an existing table directory from the filesystem. */
private void removeTablePath() throws IOException {
Configuration conf = new Configuration();
conf.set("fs.default.name", "file:///");
@ -346,7 +348,8 @@ private void assertColMinAndMax(String colName, ColumnGenerator generator)
int minId = getMinRowId();
int maxId = getMaxRowId();
LOG.info("Checking min/max for column " + colName + " with type " + generator.getType());
LOG.info("Checking min/max for column " + colName + " with type "
+ generator.getType());
String expectedMin = generator.getVerifyText(minId);
String expectedMax = generator.getVerifyText(maxId);
@ -414,7 +417,7 @@ public void testEmptyExport() throws IOException, SQLException {
multiFileTest(1, 0, 1);
}
/** Export 10 rows, make sure they load in correctly */
/** Export 10 rows, make sure they load in correctly. */
public void testTextExport() throws IOException, SQLException {
multiFileTest(1, 10, 1);
}
@ -427,7 +430,7 @@ public void testMultiFilesOneMapper() throws IOException, SQLException {
}
/** Make sure we can use CombineFileInputFormat to handle multiple
* files and multiple maps
* files and multiple maps.
*/
public void testMultiFilesMultiMaps() throws IOException, SQLException {
multiFileTest(2, 10, 2);
@ -481,7 +484,7 @@ public void testUnlimitedTransactionSize() throws IOException, SQLException {
verifyExport(TOTAL_RECORDS);
}
/** Run 2 mappers, make sure all records load in correctly */
/** Run 2 mappers, make sure all records load in correctly. */
public void testMultiMapTextExport() throws IOException, SQLException {
final int RECORDS_PER_MAP = 10;
@ -496,12 +499,13 @@ public void testMultiMapTextExport() throws IOException, SQLException {
verifyExport(RECORDS_PER_MAP * NUM_FILES);
}
/** Export some rows from a SequenceFile, make sure they import correctly */
/** Export some rows from a SequenceFile, make sure they import correctly. */
public void testSequenceFileExport() throws Exception {
final int TOTAL_RECORDS = 10;
// First, generate class and jar files that represent the table we're exporting to.
// First, generate class and jar files that represent the table
// we're exporting to.
LOG.info("Creating initial schema for SeqFile test");
createTable();
LOG.info("Generating code...");
@ -531,7 +535,8 @@ public void testSequenceFileExport() throws Exception {
String jarBaseName = jarPath.getName();
assertTrue(jarBaseName.endsWith(".jar"));
assertTrue(jarBaseName.length() > ".jar".length());
String className = jarBaseName.substring(0, jarBaseName.length() - ".jar".length());
String className = jarBaseName.substring(0, jarBaseName.length()
- ".jar".length());
LOG.info("Using jar filename: " + jarFileName);
LOG.info("Using class name: " + className);
@ -621,7 +626,7 @@ protected String pad(int n) {
}
/**
* Get a column generator for DATE columns
* Get a column generator for DATE columns.
*/
protected ColumnGenerator getDateColumnGenerator() {
return new ColumnGenerator() {

9
src/test/org/apache/hadoop/sqoop/TestMultiCols.java
View File

@ -31,10 +31,11 @@
*/
public class TestMultiCols extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog(TestMultiCols.class.getName());
public static final Log LOG = LogFactory.getLog(
TestMultiCols.class.getName());
/**
* Do a full import verification test on a table containing one row
* Do a full import verification test on a table containing one row.
* @param types the types of the columns to insert
* @param insertVals the SQL text to use to insert each value
* @param validateVals the text to expect when retrieving each value from
@ -44,12 +45,12 @@ public class TestMultiCols extends ImportJobTestCase {
* @param importColumns The list of columns to import
*/
private void verifyTypes(String [] types , String [] insertVals,
String validateVals [], String validateLine) {
String [] validateVals, String validateLine) {
verifyTypes(types, insertVals, validateVals, validateLine, null);
}
private void verifyTypes(String [] types , String [] insertVals,
String validateVals [], String validateLine, String [] importColumns) {
String [] validateVals, String validateLine, String [] importColumns) {
createTableWithColTypes(types, insertVals);

24
src/test/org/apache/hadoop/sqoop/TestMultiMaps.java
View File

@ -47,7 +47,7 @@
public class TestMultiMaps extends ImportJobTestCase {
/**
* Create the argv to pass to Sqoop
* Create the argv to pass to Sqoop.
* @return the argv as an array of strings.
*/
protected String [] getArgv(boolean includeHadoopFlags, String [] colNames,
@ -103,7 +103,7 @@ protected List<Path> getDataFilePaths() throws IOException {
}
/**
* Given a comma-delimited list of integers, grab and parse the first int
* Given a comma-delimited list of integers, grab and parse the first int.
* @param str a comma-delimited list of values, the first of which is an int.
* @return the first field in the string, cast to int
*/
@ -130,13 +130,15 @@ public void runMultiMapTest(String splitByCol, int expectedSum)
CompilationManager compileMgr = new CompilationManager(opts);
String jarFileName = compileMgr.getJarFilename();
prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());
prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
getTableName());
List<Path> paths = getDataFilePaths();
Configuration conf = new Configuration();
int curSum = 0;
assertTrue("Found only " + paths.size() + " path(s); expected > 1.", paths.size() > 1);
assertTrue("Found only " + paths.size() + " path(s); expected > 1.",
paths.size() > 1);
// We expect multiple files. We need to open all the files and sum up the
// first column across all of them.
@ -147,11 +149,12 @@ public void runMultiMapTest(String splitByCol, int expectedSum)
Object key = ReflectionUtils.newInstance(reader.getKeyClass(), conf);
Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);
// We know that these values are two ints separated by a ',' character.
// Since this is all dynamic, though, we don't want to actually link against
// the class and use its methods. So we just parse this back into int fields manually.
// Sum them up and ensure that we get the expected total for the first column, to
// verify that we got all the results from the db into the file.
// We know that these values are two ints separated by a ','
// character. Since this is all dynamic, though, we don't want to
// actually link against the class and use its methods. So we just
// parse this back into int fields manually. Sum them up and ensure
// that we get the expected total for the first column, to verify that
// we got all the results from the db into the file.
// now sum up everything in the file.
while (reader.next(key) != null) {
@ -163,7 +166,8 @@ public void runMultiMapTest(String splitByCol, int expectedSum)
reader = null;
}
assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
assertEquals("Total sum of first db column mismatch", expectedSum,
curSum);
} catch (InvalidOptionsException ioe) {
fail(ioe.toString());
} catch (ParseException pe) {

21
src/test/org/apache/hadoop/sqoop/TestSplitBy.java
View File

@ -37,12 +37,12 @@
import org.apache.hadoop.sqoop.util.ClassLoaderStack;
/**
* Test that --split-by works
* Test that --split-by works.
*/
public class TestSplitBy extends ImportJobTestCase {
/**
* Create the argv to pass to Sqoop
* Create the argv to pass to Sqoop.
* @return the argv as an array of strings.
*/
protected String [] getArgv(boolean includeHadoopFlags, String [] colNames,
@ -82,7 +82,7 @@ protected String getTableName() {
/**
* Given a comma-delimited list of integers, grab and parse the first int
* Given a comma-delimited list of integers, grab and parse the first int.
* @param str a comma-delimited list of values, the first of which is an int.
* @return the first field in the string, cast to int
*/
@ -109,7 +109,8 @@ public void runSplitByTest(String splitByCol, int expectedSum)
String jarFileName = compileMgr.getJarFilename();
LOG.debug("Got jar from import job: " + jarFileName);
prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());
prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
getTableName());
reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());
@ -119,10 +120,11 @@ public void runSplitByTest(String splitByCol, int expectedSum)
Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);
// We know that these values are two ints separated by a ',' character.
// Since this is all dynamic, though, we don't want to actually link against
// the class and use its methods. So we just parse this back into int fields manually.
// Sum them up and ensure that we get the expected total for the first column, to
// verify that we got all the results from the db into the file.
// Since this is all dynamic, though, we don't want to actually link
// against the class and use its methods. So we just parse this back
// into int fields manually. Sum them up and ensure that we get the
// expected total for the first column, to verify that we got all the
// results from the db into the file.
// Sum up everything in the file.
int curSum = 0;
@ -131,7 +133,8 @@ public void runSplitByTest(String splitByCol, int expectedSum)
curSum += getFirstInt(val.toString());
}
assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
assertEquals("Total sum of first db column mismatch", expectedSum,
curSum);
} catch (InvalidOptionsException ioe) {
fail(ioe.toString());
} catch (ParseException pe) {

31
src/test/org/apache/hadoop/sqoop/TestSqoopOptions.java
View File

@ -24,7 +24,7 @@
/**
* Test aspects of the SqoopOptions class
* Test aspects of the SqoopOptions class.
*/
public class TestSqoopOptions extends TestCase {
@ -158,8 +158,8 @@ private SqoopOptions parse(String [] argv) throws Exception {
// test that setting output delimiters also sets input delimiters
public void testDelimitersInherit() throws Exception {
String [] args = {
"--fields-terminated-by",
"|"
"--fields-terminated-by",
"|",
};
SqoopOptions opts = parse(args);
@ -167,13 +167,14 @@ public void testDelimitersInherit() throws Exception {
assertEquals('|', opts.getOutputFieldDelim());
}
// test that setting output delimiters and setting input delims separately works
// Test that setting output delimiters and setting input delims
// separately works.
public void testDelimOverride1() throws Exception {
String [] args = {
"--fields-terminated-by",
"|",
"--input-fields-terminated-by",
"*"
"--fields-terminated-by",
"|",
"--input-fields-terminated-by",
"*",
};
SqoopOptions opts = parse(args);
@ -184,10 +185,10 @@ public void testDelimOverride1() throws Exception {
// test that the order in which delims are specified doesn't matter
public void testDelimOverride2() throws Exception {
String [] args = {
"--input-fields-terminated-by",
"*",
"--fields-terminated-by",
"|"
"--input-fields-terminated-by",
"*",
"--fields-terminated-by",
"|",
};
SqoopOptions opts = parse(args);
@ -198,7 +199,7 @@ public void testDelimOverride2() throws Exception {
public void testBadNumMappers1() throws Exception {
String [] args = {
"--num-mappers",
"x"
"x",
};
try {
@ -212,7 +213,7 @@ public void testBadNumMappers1() throws Exception {
public void testBadNumMappers2() throws Exception {
String [] args = {
"-m",
"x"
"x",
};
try {
@ -226,7 +227,7 @@ public void testBadNumMappers2() throws Exception {
public void testGoodNumMappers() throws Exception {
String [] args = {
"-m",
"4"
"4",
};
SqoopOptions opts = parse(args);

29
src/test/org/apache/hadoop/sqoop/TestWhere.java
View File

@ -45,7 +45,7 @@
public class TestWhere extends ImportJobTestCase {
/**
* Create the argv to pass to Sqoop
* Create the argv to pass to Sqoop.
* @return the argv as an array of strings.
*/
protected String [] getArgv(boolean includeHadoopFlags, String [] colNames,
@ -87,7 +87,7 @@ protected String getTableName() {
/**
* Given a comma-delimited list of integers, grab and parse the first int
* Given a comma-delimited list of integers, grab and parse the first int.
* @param str a comma-delimited list of values, the first of which is an int.
* @return the first field in the string, cast to int
*/
@ -96,8 +96,8 @@ private int getFirstInt(String str) {
return Integer.parseInt(parts[0]);
}
public void runWhereTest(String whereClause, String firstValStr, int numExpectedResults,
int expectedSum) throws IOException {
public void runWhereTest(String whereClause, String firstValStr,
int numExpectedResults, int expectedSum) throws IOException {
String [] columns = HsqldbTestServer.getFieldNames();
ClassLoader prevClassLoader = null;
@ -113,7 +113,8 @@ public void runWhereTest(String whereClause, String firstValStr, int numExpected
CompilationManager compileMgr = new CompilationManager(opts);
String jarFileName = compileMgr.getJarFilename();
prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());
prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
getTableName());
reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());
@ -128,13 +129,15 @@ public void runWhereTest(String whereClause, String firstValStr, int numExpected
// make sure that the value we think should be at the top, is.
reader.getCurrentValue(val);
assertEquals("Invalid ordering within sorted SeqFile", firstValStr, val.toString());
assertEquals("Invalid ordering within sorted SeqFile", firstValStr,
val.toString());
// We know that these values are two ints separated by a ',' character.
// Since this is all dynamic, though, we don't want to actually link against
// the class and use its methods. So we just parse this back into int fields manually.
// Sum them up and ensure that we get the expected total for the first column, to
// verify that we got all the results from the db into the file.
// Since this is all dynamic, though, we don't want to actually link
// against the class and use its methods. So we just parse this back
// into int fields manually. Sum them up and ensure that we get the
// expected total for the first column, to verify that we got all the
// results from the db into the file.
int curSum = getFirstInt(val.toString());
int totalResults = 1;
@ -145,8 +148,10 @@ public void runWhereTest(String whereClause, String firstValStr, int numExpected
totalResults++;
}
assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
assertEquals("Incorrect number of results for query", numExpectedResults, totalResults);
assertEquals("Total sum of first db column mismatch", expectedSum,
curSum);
assertEquals("Incorrect number of results for query", numExpectedResults,
totalResults);
} catch (InvalidOptionsException ioe) {
fail(ioe.toString());
} catch (ParseException pe) {

7
src/test/org/apache/hadoop/sqoop/ThirdPartyTests.java
View File

@ -33,9 +33,10 @@
import org.apache.hadoop.sqoop.manager.PostgresqlTest;
/**
* Test battery including all tests of vendor-specific ConnManager implementations.
* These tests likely aren't run by Apache Hudson, because they require configuring
* and using Oracle, MySQL, etc., which may have incompatible licenses with Apache.
* Test battery including all tests of vendor-specific ConnManager
* implementations. These tests likely aren't run by Apache Hudson, because
* they require configuring and using Oracle, MySQL, etc., which may have
* incompatible licenses with Apache.
*/
public final class ThirdPartyTests extends TestCase {

36
src/test/org/apache/hadoop/sqoop/hive/TestHiveImport.java
View File

@ -42,7 +42,8 @@
*/
public class TestHiveImport extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog(TestHiveImport.class.getName());
public static final Log LOG = LogFactory.getLog(
TestHiveImport.class.getName());
/**
* Sets the expected number of columns in the table being manipulated
@ -60,7 +61,7 @@ private void setNumCols(int numCols) {
}
/**
* Create the argv to pass to Sqoop
* Create the argv to pass to Sqoop.
* @return the argv as an array of strings.
*/
protected String [] getArgv(boolean includeHadoopFlags, String [] moreArgs) {
@ -147,8 +148,9 @@ private SqoopOptions getSqoopOptions(String [] args, SqoopTool tool) {
return opts;
}
private void runImportTest(String tableName, String [] types, String [] values,
String verificationScript, String [] args, SqoopTool tool) throws IOException {
private void runImportTest(String tableName, String [] types,
String [] values, String verificationScript, String [] args,
SqoopTool tool) throws IOException {
// create a table and populate it with a row...
createTableWithColTypes(types, values);
@ -158,7 +160,8 @@ private void runImportTest(String tableName, String [] types, String [] values,
SqoopOptions options = getSqoopOptions(args, tool);
String hiveHome = options.getHiveHome();
assertNotNull("hive.home was not set", hiveHome);
Path testDataPath = new Path(new Path(hiveHome), "scripts/" + verificationScript);
Path testDataPath = new Path(new Path(hiveHome),
"scripts/" + verificationScript);
System.setProperty("expected.script", testDataPath.toString());
// verify that we can import it correctly into hive.
@ -202,7 +205,7 @@ public void testGenerateOnly() throws IOException {
}
/** Test that strings and ints are handled in the normal fashion */
/** Test that strings and ints are handled in the normal fashion. */
@Test
public void testNormalHiveImport() throws IOException {
final String TABLE_NAME = "NORMAL_HIVE_IMPORT";
@ -214,7 +217,7 @@ public void testNormalHiveImport() throws IOException {
getArgv(false, null), new ImportTool());
}
/** Test that table is created in hive with no data import */
/** Test that table is created in hive with no data import. */
@Test
public void testCreateOnlyHiveImport() throws IOException {
final String TABLE_NAME = "CREATE_ONLY_HIVE_IMPORT";
@ -227,7 +230,10 @@ public void testCreateOnlyHiveImport() throws IOException {
new CreateHiveTableTool());
}
/** Test that table is created in hive and replaces the existing table if any */
/**
* Test that table is created in hive and replaces the existing table if
* any.
*/
@Test
public void testCreateOverwriteHiveImport() throws IOException {
final String TABLE_NAME = "CREATE_OVERWRITE_HIVE_IMPORT";
@ -241,7 +247,7 @@ public void testCreateOverwriteHiveImport() throws IOException {
new CreateHiveTableTool());
}
/** Test that dates are coerced properly to strings */
/** Test that dates are coerced properly to strings. */
@Test
public void testDate() throws IOException {
final String TABLE_NAME = "DATE_HIVE_IMPORT";
@ -253,7 +259,7 @@ public void testDate() throws IOException {
getArgv(false, null), new ImportTool());
}
/** Test that NUMERICs are coerced to doubles */
/** Test that NUMERICs are coerced to doubles. */
@Test
public void testNumeric() throws IOException {
final String TABLE_NAME = "NUMERIC_HIVE_IMPORT";
@ -265,7 +271,7 @@ public void testNumeric() throws IOException {
getArgv(false, null), new ImportTool());
}
/** If bin/hive returns an error exit status, we should get an IOException */
/** If bin/hive returns an error exit status, we should get an IOException. */
@Test
public void testHiveExitFails() {
// The expected script is different than the one which would be generated
@ -285,7 +291,7 @@ public void testHiveExitFails() {
}
}
/** Test that we can set delimiters how we want them */
/** Test that we can set delimiters how we want them. */
@Test
public void testCustomDelimiters() throws IOException {
final String TABLE_NAME = "CUSTOM_DELIM_IMPORT";
@ -293,8 +299,10 @@ public void testCustomDelimiters() throws IOException {
setNumCols(3);
String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" };
String [] vals = { "'test'", "42", "'somestring'" };
String [] extraArgs = { "--fields-terminated-by", ",",
"--lines-terminated-by", "|" };
String [] extraArgs = {
"--fields-terminated-by", ",",
"--lines-terminated-by", "|",
};
runImportTest(TABLE_NAME, types, vals, "customDelimImport.q",
getArgv(false, extraArgs), new ImportTool());
}

7
src/test/org/apache/hadoop/sqoop/hive/TestTableDefWriter.java
View File

@ -25,7 +25,6 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.sqoop.SqoopOptions;
import org.junit.Test;
import junit.framework.TestCase;
@ -34,7 +33,8 @@
*/
public class TestTableDefWriter extends TestCase {
public static final Log LOG = LogFactory.getLog(TestTableDefWriter.class.getName());
public static final Log LOG = LogFactory.getLog(
TestTableDefWriter.class.getName());
// Test getHiveOctalCharCode and expect an IllegalArgumentException.
private void expectExceptionInCharCode(int charCode) {
@ -73,7 +73,8 @@ public void testDifferentTableNames() throws Exception {
LOG.debug("Load data stmt: " + loadData);
// Assert that the statements generated have the form we expect.
assertTrue(createTable.indexOf("CREATE TABLE IF NOT EXISTS outputTable") != -1);
assertTrue(createTable.indexOf(
"CREATE TABLE IF NOT EXISTS outputTable") != -1);
assertTrue(loadData.indexOf("INTO TABLE outputTable") != -1);
assertTrue(loadData.indexOf("/inputTable'") != -1);
}

Some files were not shown because too many files have changed in this diff Show More