split(Configuration conf, ResultSet results,
String lowClausePrefix = colName + " >= ";
String highClausePrefix = colName + " < ";
-
+
BigDecimal numSplits = new BigDecimal(
- HadoopShim.get().getConfNumMaps(conf));
+ ConfigurationHelper.getConfNumMaps(conf));
if (minVal == null && maxVal == null) {
// Range is null to null. Return a null split accordingly.
diff --git a/src/java/com/cloudera/sqoop/mapreduce/db/DBInputFormat.java b/src/java/com/cloudera/sqoop/mapreduce/db/DBInputFormat.java
index 523fd9a1..cff6110d 100644
--- a/src/java/com/cloudera/sqoop/mapreduce/db/DBInputFormat.java
+++ b/src/java/com/cloudera/sqoop/mapreduce/db/DBInputFormat.java
@@ -42,15 +42,15 @@
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-import com.cloudera.sqoop.shims.HadoopShim;
+import com.cloudera.sqoop.config.ConfigurationHelper;
/**
* A InputFormat that reads input data from an SQL table.
*
- * DBInputFormat emits LongWritables containing the record number as
- * key and DBWritables as value.
- *
- * The SQL query, and input class can be using one of the two
+ * DBInputFormat emits LongWritables containing the record number as
+ * key and DBWritables as value.
+ *
+ * The SQL query, and input class can be using one of the two
* setInput methods.
*/
public class DBInputFormat
@@ -71,7 +71,7 @@ public void write(DataOutput out) throws IOException { }
@Override
public void write(PreparedStatement arg0) throws SQLException { }
}
-
+
/**
* A InputSplit that spans a set of rows.
*/
@@ -172,7 +172,7 @@ public void setConf(Configuration conf) {
public Configuration getConf() {
return dbConf.getConf();
}
-
+
public DBConfiguration getDBConf() {
return dbConf;
}
@@ -227,7 +227,7 @@ conf, getConnection(), getDBConf(), conditions, fieldNames,
@Override
/** {@inheritDoc} */
public RecordReader createRecordReader(InputSplit split,
- TaskAttemptContext context) throws IOException, InterruptedException {
+ TaskAttemptContext context) throws IOException, InterruptedException {
return createDBRecordReader((DBInputSplit) split,
context.getConfiguration());
@@ -237,7 +237,7 @@ public RecordReader createRecordReader(InputSplit split,
@Override
public List getSplits(JobContext job) throws IOException {
- ResultSet results = null;
+ ResultSet results = null;
Statement statement = null;
try {
statement = connection.createStatement();
@@ -246,7 +246,7 @@ public List getSplits(JobContext job) throws IOException {
results.next();
long count = results.getLong(1);
- int chunks = HadoopShim.get().getJobNumMaps(job);
+ int chunks = ConfigurationHelper.getJobNumMaps(job);
long chunkSize = (count / chunks);
results.close();
@@ -285,14 +285,14 @@ public List getSplits(JobContext job) throws IOException {
}
}
- /** Returns the query for getting the total number of rows,
+ /** Returns the query for getting the total number of rows,
* subclasses can override this for custom behaviour.*/
protected String getCountQuery() {
-
+
if(dbConf.getInputCountQuery() != null) {
return dbConf.getInputCountQuery();
}
-
+
StringBuilder query = new StringBuilder();
query.append("SELECT COUNT(*) FROM " + tableName);
@@ -304,20 +304,20 @@ protected String getCountQuery() {
/**
* Initializes the map-part of the job with the appropriate input settings.
- *
+ *
* @param job The map-reduce job
- * @param inputClass the class object implementing DBWritable, which is the
+ * @param inputClass the class object implementing DBWritable, which is the
* Java object holding tuple fields.
* @param tableName The table to read data from
- * @param conditions The condition which to select data with,
+ * @param conditions The condition which to select data with,
* eg. '(updated > 20070101 AND length > 0)'
* @param orderBy the fieldNames in the orderBy clause.
* @param fieldNames The field names in the table
* @see #setInput(Job, Class, String, String)
*/
- public static void setInput(Job job,
+ public static void setInput(Job job,
Class inputClass,
- String tableName, String conditions,
+ String tableName, String conditions,
String orderBy, String... fieldNames) {
job.setInputFormatClass(DBInputFormat.class);
DBConfiguration dbConf = new DBConfiguration(job.getConfiguration());
@@ -327,17 +327,17 @@ public static void setInput(Job job,
dbConf.setInputConditions(conditions);
dbConf.setInputOrderBy(orderBy);
}
-
+
/**
* Initializes the map-part of the job with the appropriate input settings.
- *
+ *
* @param job The map-reduce job
- * @param inputClass the class object implementing DBWritable, which is the
+ * @param inputClass the class object implementing DBWritable, which is the
* Java object holding tuple fields.
- * @param inputQuery the input query to select fields. Example :
+ * @param inputQuery the input query to select fields. Example :
* "SELECT f1, f2, f3 FROM Mytable ORDER BY f1"
- * @param inputCountQuery the input query that returns
- * the number of records in the table.
+ * @param inputCountQuery the input query that returns
+ * the number of records in the table.
* Example : "SELECT COUNT(f1) FROM Mytable"
* @see #setInput(Job, Class, String, String, String, String...)
*/
diff --git a/src/java/com/cloudera/sqoop/mapreduce/db/DBOutputFormat.java b/src/java/com/cloudera/sqoop/mapreduce/db/DBOutputFormat.java
index cc022c07..a74f9b20 100644
--- a/src/java/com/cloudera/sqoop/mapreduce/db/DBOutputFormat.java
+++ b/src/java/com/cloudera/sqoop/mapreduce/db/DBOutputFormat.java
@@ -36,24 +36,24 @@
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.StringUtils;
-import com.cloudera.sqoop.shims.HadoopShim;
+import com.cloudera.sqoop.config.ConfigurationHelper;
/**
* A OutputFormat that sends the reduce output to a SQL table.
- *
- * {@link DBOutputFormat} accepts <key,value> pairs, where
- * key has a type extending DBWritable. Returned {@link RecordWriter}
- * writes only the key to the database with a batch SQL query.
- *
+ *
+ * {@link DBOutputFormat} accepts <key,value> pairs, where
+ * key has a type extending DBWritable. Returned {@link RecordWriter}
+ * writes only the key to the database with a batch SQL query.
+ *
*/
-public class DBOutputFormat
+public class DBOutputFormat
extends OutputFormat {
private static final Log LOG = LogFactory.getLog(DBOutputFormat.class);
- public void checkOutputSpecs(JobContext context)
+ public void checkOutputSpecs(JobContext context)
throws IOException, InterruptedException {}
- public OutputCommitter getOutputCommitter(TaskAttemptContext context)
+ public OutputCommitter getOutputCommitter(TaskAttemptContext context)
throws IOException, InterruptedException {
return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
context);
@@ -62,7 +62,7 @@ public OutputCommitter getOutputCommitter(TaskAttemptContext context)
/**
* A RecordWriter that writes the reduce output to a SQL table.
*/
- public class DBRecordWriter
+ public class DBRecordWriter
extends RecordWriter {
private Connection connection;
@@ -81,11 +81,11 @@ public DBRecordWriter(Connection connection
public Connection getConnection() {
return connection;
}
-
+
public PreparedStatement getStatement() {
return statement;
}
-
+
@Override
/** {@inheritDoc} */
public void close(TaskAttemptContext context) throws IOException {
@@ -123,7 +123,7 @@ public void write(K key, V value) throws IOException {
/**
* Constructs the query used as the prepared statement to insert data.
- *
+ *
* @param table
* the table to insert into
* @param fieldNames
@@ -163,20 +163,20 @@ public String constructQuery(String table, String[] fieldNames) {
@Override
/** {@inheritDoc} */
- public RecordWriter getRecordWriter(TaskAttemptContext context)
+ public RecordWriter getRecordWriter(TaskAttemptContext context)
throws IOException {
DBConfiguration dbConf = new DBConfiguration(context.getConfiguration());
String tableName = dbConf.getOutputTableName();
String[] fieldNames = dbConf.getOutputFieldNames();
-
+
if(fieldNames == null) {
fieldNames = new String[dbConf.getOutputFieldCount()];
}
-
+
try {
Connection connection = dbConf.getConnection();
PreparedStatement statement = null;
-
+
statement = connection.prepareStatement(
constructQuery(tableName, fieldNames));
return new DBRecordWriter(connection, statement);
@@ -188,12 +188,12 @@ public RecordWriter getRecordWriter(TaskAttemptContext context)
/**
* Initializes the reduce-part of the job with
* the appropriate output settings.
- *
+ *
* @param job The job
* @param tableName The table to insert data into
* @param fieldNames The field names in the table.
*/
- public static void setOutput(Job job, String tableName,
+ public static void setOutput(Job job, String tableName,
String... fieldNames) throws IOException {
if(fieldNames.length > 0 && fieldNames[0] != null) {
DBConfiguration dbConf = setOutput(job, tableName);
@@ -201,34 +201,34 @@ public static void setOutput(Job job, String tableName,
} else {
if (fieldNames.length > 0) {
setOutput(job, tableName, fieldNames.length);
- } else {
+ } else {
throw new IllegalArgumentException(
"Field names must be greater than 0");
}
}
}
-
+
/**
- * Initializes the reduce-part of the job
+ * Initializes the reduce-part of the job
* with the appropriate output settings.
- *
+ *
* @param job The job
* @param tableName The table to insert data into
* @param fieldCount the number of fields in the table.
*/
- public static void setOutput(Job job, String tableName,
+ public static void setOutput(Job job, String tableName,
int fieldCount) throws IOException {
DBConfiguration dbConf = setOutput(job, tableName);
dbConf.setOutputFieldCount(fieldCount);
}
-
+
private static DBConfiguration setOutput(Job job,
String tableName) throws IOException {
job.setOutputFormatClass(DBOutputFormat.class);
- HadoopShim.get().setJobReduceSpeculativeExecution(job, false);
+ ConfigurationHelper.setJobReduceSpeculativeExecution(job, false);
DBConfiguration dbConf = new DBConfiguration(job.getConfiguration());
-
+
dbConf.setOutputTableName(tableName);
return dbConf;
}
diff --git a/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java b/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
index e482edb0..6708ceed 100644
--- a/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
+++ b/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
@@ -42,7 +42,7 @@
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-import com.cloudera.sqoop.shims.HadoopShim;
+import com.cloudera.sqoop.config.ConfigurationHelper;
/**
* A InputFormat that reads input data from an SQL table.
@@ -167,7 +167,7 @@ protected DBSplitter getSplitter(int sqlDataType) {
/** {@inheritDoc} */
public List getSplits(JobContext job) throws IOException {
- int targetNumTasks = HadoopShim.get().getJobNumMaps(job);
+ int targetNumTasks = ConfigurationHelper.getJobNumMaps(job);
if (1 == targetNumTasks) {
// There's no need to run a bounding vals query; just return a split
// that separates nothing. This can be considerably more optimal for a
@@ -313,9 +313,9 @@ conf, getConnection(), dbConf, dbConf.getInputConditions(),
* We reuse the same field, but it's not strictly ordering it
* -- just partitioning the results.
*/
- public static void setInput(Job job,
+ public static void setInput(Job job,
Class inputClass,
- String tableName, String conditions,
+ String tableName, String conditions,
String splitBy, String... fieldNames) {
DBInputFormat.setInput(job, inputClass, tableName, conditions,
splitBy, fieldNames);
diff --git a/src/java/com/cloudera/sqoop/mapreduce/db/DateSplitter.java b/src/java/com/cloudera/sqoop/mapreduce/db/DateSplitter.java
index d6edc9f7..d4830d3d 100644
--- a/src/java/com/cloudera/sqoop/mapreduce/db/DateSplitter.java
+++ b/src/java/com/cloudera/sqoop/mapreduce/db/DateSplitter.java
@@ -27,11 +27,10 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
-import com.cloudera.sqoop.shims.HadoopShim;
+import com.cloudera.sqoop.config.ConfigurationHelper;
/**
* Implement DBSplitter over date/time values.
@@ -55,7 +54,7 @@ public List split(Configuration conf, ResultSet results,
String lowClausePrefix = colName + " >= ";
String highClausePrefix = colName + " < ";
- int numSplits = HadoopShim.get().getConfNumMaps(conf);
+ int numSplits = ConfigurationHelper.getConfNumMaps(conf);
if (numSplits < 1) {
numSplits = 1;
}
diff --git a/src/java/com/cloudera/sqoop/mapreduce/db/FloatSplitter.java b/src/java/com/cloudera/sqoop/mapreduce/db/FloatSplitter.java
index 65c87038..1023bc2c 100644
--- a/src/java/com/cloudera/sqoop/mapreduce/db/FloatSplitter.java
+++ b/src/java/com/cloudera/sqoop/mapreduce/db/FloatSplitter.java
@@ -29,7 +29,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
-import com.cloudera.sqoop.shims.HadoopShim;
+import com.cloudera.sqoop.config.ConfigurationHelper;
/**
* Implement DBSplitter over floating-point values.
@@ -62,7 +62,7 @@ public List split(Configuration conf, ResultSet results,
// Use this as a hint. May need an extra task if the size doesn't
// divide cleanly.
- int numSplits = HadoopShim.get().getConfNumMaps(conf);
+ int numSplits = ConfigurationHelper.getConfNumMaps(conf);
double splitSize = (maxVal - minVal) / (double) numSplits;
if (splitSize < MIN_INCREMENT) {
diff --git a/src/java/com/cloudera/sqoop/mapreduce/db/IntegerSplitter.java b/src/java/com/cloudera/sqoop/mapreduce/db/IntegerSplitter.java
index bbafa780..cd3b5ece 100644
--- a/src/java/com/cloudera/sqoop/mapreduce/db/IntegerSplitter.java
+++ b/src/java/com/cloudera/sqoop/mapreduce/db/IntegerSplitter.java
@@ -26,7 +26,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
-import com.cloudera.sqoop.shims.HadoopShim;
+import com.cloudera.sqoop.config.ConfigurationHelper;
/**
* Implement DBSplitter over integer values.
@@ -41,7 +41,7 @@ public List split(Configuration conf, ResultSet results,
String lowClausePrefix = colName + " >= ";
String highClausePrefix = colName + " < ";
- int numSplits = HadoopShim.get().getConfNumMaps(conf);
+ int numSplits = ConfigurationHelper.getConfNumMaps(conf);
if (numSplits < 1) {
numSplits = 1;
}
diff --git a/src/java/com/cloudera/sqoop/mapreduce/db/TextSplitter.java b/src/java/com/cloudera/sqoop/mapreduce/db/TextSplitter.java
index 09afad41..79aa57c2 100644
--- a/src/java/com/cloudera/sqoop/mapreduce/db/TextSplitter.java
+++ b/src/java/com/cloudera/sqoop/mapreduce/db/TextSplitter.java
@@ -18,19 +18,18 @@
package com.cloudera.sqoop.mapreduce.db;
+import java.math.BigDecimal;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
-import com.cloudera.sqoop.shims.HadoopShim;
+import com.cloudera.sqoop.config.ConfigurationHelper;
/**
* Implement DBSplitter over text strings.
@@ -88,7 +87,7 @@ public List split(Configuration conf, ResultSet results,
// Use this as a hint. May need an extra task if the size doesn't
// divide cleanly.
- int numSplits = HadoopShim.get().getConfNumMaps(conf);
+ int numSplits = ConfigurationHelper.getConfNumMaps(conf);
String lowClausePrefix = colName + " >= '";
String highClausePrefix = colName + " < '";
diff --git a/src/java/com/cloudera/sqoop/shims/HadoopShim.java b/src/java/com/cloudera/sqoop/shims/HadoopShim.java
deleted file mode 100644
index f216666f..00000000
--- a/src/java/com/cloudera/sqoop/shims/HadoopShim.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.sqoop.shims;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.MapContext;
-
-/**
- * In order to be compatible with multiple versions of Hadoop, all parts
- * of the Hadoop interface that are not cross-version compatible are
- * encapsulated in an implementation of this class. Users should use
- * the ShimLoader class as a factory to obtain an implementation of
- * HadoopShims corresponding to the version of Hadoop currently on the
- * classpath.
- */
-public abstract class HadoopShim {
-
- protected HadoopShim() {
- }
-
- /**
- * Parse arguments in 'args' via the GenericOptionsParser and
- * embed the results in the supplied configuration.
- * @param conf the configuration to populate with generic options.
- * @param args the arguments to process.
- * @return the unused args to be passed to the application itself.
- */
- public abstract String [] parseGenericOptions(Configuration conf,
- String [] args) throws IOException;
-
- /**
- * @return the number of mapper output records from a job using its counters.
- */
- public abstract long getNumMapOutputRecords(Job job)
- throws IOException, InterruptedException;
-
- /**
- * @return the number of mapper input records from a job using its counters.
- */
- public abstract long getNumMapInputRecords(Job job)
- throws IOException, InterruptedException;
-
- /**
- * @return the Configuration property identifying the current task id.
- */
- public abstract String getTaskIdProperty();
-
- /**
- * @return the Configuration property identifying the job's local dir.
- */
- public abstract String getJobLocalDirProperty();
-
- /**
- * Set the (hinted) number of map tasks for a job.
- */
- public abstract void setJobNumMaps(Job job, int numMapTasks);
-
- /**
- * Get the (hinted) number of map tasks for a job.
- */
- public abstract int getJobNumMaps(JobContext job);
-
- /**
- * Get the (hinted) number of map tasks for a job.
- */
- public abstract int getConfNumMaps(Configuration conf);
-
- /**
- * Set the mapper speculative execution property for a job.
- */
- public abstract void setJobMapSpeculativeExecution(Job job,
- boolean isEnabled);
-
- /**
- * Set the reducer speculative execution property for a job.
- */
- public abstract void setJobReduceSpeculativeExecution(Job job,
- boolean isEnabled);
-
- /**
- * Sets the Jobtracker address to use for a job.
- */
- public abstract void setJobtrackerAddr(Configuration conf, String addr);
-
- /**
- * Returns the Configuration property identifying a DBWritable to use.
- */
- public abstract String getDbInputClassProperty();
-
- /**
- * Returns the Configuration property identifying the DB username.
- */
- public abstract String getDbUsernameProperty();
-
- /**
- * Returns the Configuration property identifying the DB password.
- */
- public abstract String getDbPasswordProperty();
-
- /**
- * Returns the Configuration property identifying the DB connect string.
- */
- public abstract String getDbUrlProperty();
-
- /**
- * Returns the Configuration property identifying the DB input table.
- */
- public abstract String getDbInputTableNameProperty();
-
- /**
- * Returns the Configuration property specifying WHERE conditions for the
- * db table.
- */
- public abstract String getDbInputConditionsProperty();
-
- /**
- * Returns a mock MapContext that has both an OutputCommitter and an
- * InputSplit wired to the specified path.
- * Used for testing LargeObjectLoader.
- */
- public abstract MapContext getMapContextForIOPath(
- Configuration conf, Path p);
-
- public static final synchronized HadoopShim get() {
- return ShimLoader.getHadoopShim(null);
- }
-}
diff --git a/src/java/com/cloudera/sqoop/shims/ShimLoader.java b/src/java/com/cloudera/sqoop/shims/ShimLoader.java
deleted file mode 100644
index 4ef62842..00000000
--- a/src/java/com/cloudera/sqoop/shims/ShimLoader.java
+++ /dev/null
@@ -1,266 +0,0 @@
-/**
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.sqoop.shims;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.VersionInfo;
-import com.cloudera.sqoop.util.ClassLoaderStack;
-
-/**
- * Provides a service locator for the appropriate shim, dynamically chosen
- * based on the Hadoop version in the classpath.
- */
-public abstract class ShimLoader {
- private static HadoopShim hadoopShim;
-
- public static final Log LOG = LogFactory.getLog(ShimLoader.class.getName());
-
- /**
- * Which directory Sqoop checks for shim jars.
- */
- public static final String SHIM_JAR_DIR_PROPERTY = "sqoop.shim.jar.dir";
-
- /**
- * The names of the classes for shimming Hadoop.
- * This list must be maintained in the same order as HADOOP_SHIM_MATCHES
- */
- private static final List HADOOP_SHIM_CLASSES =
- new ArrayList();
-
- /**
- * Patterns to match to identify which shim jar to load when shimming
- * Hadoop.
- * This list must be maintained in the same order as HADOOP_SHIM_MATCHES
- */
- private static final List HADOOP_SHIM_JARS =
- new ArrayList();
-
- /**
- * The regular expressions compared against the Hadoop version string
- * when determining which shim class to load.
- */
- private static final List HADOOP_SHIM_MATCHES =
- new ArrayList();
-
- static {
- // These regular expressions will be evaluated in order until one matches.
-
- // CDH3 distribution has versions as follows (with nnn being build number):
- // 0.20.2+nnn: for CDH3B2 release
- // 0.20.3-CDH3-SNAPSHOT: for CDH3B3 development builds
- // 0.20.3+nnn: expected for CDH3B3 release
- HADOOP_SHIM_MATCHES.add("0.20.(2|3)((\\+[0-9]+)|(-CDH3-SNAPSHOT))");
- HADOOP_SHIM_CLASSES.add("com.cloudera.sqoop.shims.CDH3Shim");
- HADOOP_SHIM_JARS.add("sqoop-shim-cloudera-.*.jar");
-
- // Apache 0.22 trunk.
- // Version may have the form "0.22-SNAPSHOT"
- HADOOP_SHIM_MATCHES.add("0.22-.*");
- HADOOP_SHIM_CLASSES.add("com.cloudera.sqoop.shims.Apache22HadoopShim");
- HADOOP_SHIM_JARS.add("sqoop-shim-apache-.*.jar");
-
- // ... or "0.22.n-SNAPSHOT"
- HADOOP_SHIM_MATCHES.add("0.22.\\d+-.*");
- HADOOP_SHIM_CLASSES.add("com.cloudera.sqoop.shims.Apache22HadoopShim");
- HADOOP_SHIM_JARS.add("sqoop-shim-apache-.*.jar");
-
- // Apache 0.22 trunk snapshots often compile with "Unknown" version,
- // so we default to guessing Apache in this case.
- HADOOP_SHIM_MATCHES.add("Unknown");
- HADOOP_SHIM_CLASSES.add("com.cloudera.sqoop.shims.Apache22HadoopShim");
- HADOOP_SHIM_JARS.add("sqoop-shim-apache-.*.jar");
-
- // Apache 0.21 uses the same shim jars as 0.22
- HADOOP_SHIM_MATCHES.add("0.21.\\d+(-.*)?");
- HADOOP_SHIM_CLASSES.add("com.cloudera.sqoop.shims.Apache22HadoopShim");
- HADOOP_SHIM_JARS.add("sqoop-shim-apache-.*.jar");
-
- }
-
- /**
- * Factory method to get an instance of HadoopShim based on the
- * version of Hadoop on the classpath.
- * @param conf an optional Configuration whose internal ClassLoader
- * should be updated with the jar containing the HadoopShim.
- */
- public static synchronized HadoopShim getHadoopShim(Configuration conf) {
- if (hadoopShim == null) {
- hadoopShim = loadShim(HADOOP_SHIM_MATCHES, HADOOP_SHIM_CLASSES,
- HADOOP_SHIM_JARS, HadoopShim.class, conf);
- }
- return hadoopShim;
- }
-
- /**
- * Factory method to get an instance of HadoopShim based on the
- * version of Hadoop on the classpath.
- */
- public static synchronized HadoopShim getHadoopShim() {
- return getHadoopShim(null);
- }
-
- @SuppressWarnings("unchecked")
- /**
- * Actually load the shim for the current Hadoop version.
- * @param matchExprs a list of regexes against which the current Hadoop
- * version is compared. The first one to hit defines which class/jar to
- * use.
- * @param classNames a list in the same order as matchExprs. This defines
- * what class name to load as the shim class if the Hadoop version matches
- * matchExprs[i].
- * @param jarPatterns a list in the same order as matchExprs. This defines
- * a pattern to select a jar file from which the shim classes should be
- * loaded.
- * @param xface the shim interface that the shim class must match.
- * @param conf an optional Configuration whose context classloader should
- * be updated to the current Thread's contextClassLoader after pushing a
- * new ClassLoader on the stack to load this shim jar.
- */
- private static T loadShim(List matchExprs,
- List classNames, List jarPatterns, Class xface,
- Configuration conf) {
- String version = VersionInfo.getVersion();
-
- LOG.debug("Loading shims for class : " + xface.getName());
- LOG.debug("Hadoop version: " + version);
-
- for (int i = 0; i < matchExprs.size(); i++) {
- LOG.debug("Checking: " + matchExprs.get(i));
- if (version.matches(matchExprs.get(i))) {
- String className = classNames.get(i);
- String jarPattern = jarPatterns.get(i);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Version matched regular expression: " + matchExprs.get(i));
- LOG.debug("Trying to load class: " + className);
- }
-
- // Test to see if the class is already on the classpath.
- try {
- // If we can load the shim directly, we just do so. In this case,
- // there's no need to update the Configuration's classloader,
- // because we didn't modify the classloader stack.
- return getShimInstance(className, xface);
- } catch (Exception e) {
- // Not already present. We'll need to load a jar for this.
- // Ignore this exception.
- }
-
- try {
- LOG.debug("Searching for jar matching: " + jarPattern);
- loadMatchingShimJar(jarPattern, className);
- LOG.debug("Loading shim from jar");
- T shim = getShimInstance(className, xface);
-
- if (null != conf) {
- // Set the context classloader for the base Configuration to
- // the current one, so we can load more classes from the shim jar.
- conf.setClassLoader(Thread.currentThread().getContextClassLoader());
- }
-
- return shim;
- } catch (Exception e) {
- throw new RuntimeException("Could not load shim in class "
- + className, e);
- }
- }
- }
-
- throw new RuntimeException("Could not find appropriate Hadoop shim for "
- + version);
- }
-
- /**
- * Check the current classloader to see if it can load the prescribed
- * class name as an instance of 'xface'. If so, create an instance of
- * the class and return it.
- * @param className the shim class to attempt to instantiate.
- * @param xface the interface it must implement.
- * @return an instance of className.
- */
- private static T getShimInstance(String className, Class xface)
- throws ClassNotFoundException, InstantiationException,
- IllegalAccessException {
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
- Class clazz = Class.forName(className, true, cl);
- return xface.cast(clazz.newInstance());
- }
-
- /**
- * Look through the shim directory for a jar matching 'jarPattern'
- * and classload it.
- * @param jarPattern a regular expression which the shim jar's filename
- * must match.
- * @param className a class to classload from the jar.
- */
- private static void loadMatchingShimJar(String jarPattern, String className)
- throws IOException {
- String jarFilename;
-
- String shimDirName = System.getProperty(SHIM_JAR_DIR_PROPERTY, ".");
- File shimDir = new File(shimDirName);
- if (!shimDir.exists()) {
- throw new IOException("No such shim directory: " + shimDirName);
- }
-
- String [] candidates = shimDir.list();
- if (null == candidates) {
- throw new IOException("Could not list shim directory: " + shimDirName);
- }
-
- for (String candidate : candidates) {
- if (candidate.matches(jarPattern)) {
- LOG.debug("Found jar matching pattern " + jarPattern + ": "
- + candidate);
- File jarFile = new File(shimDir, candidate);
- String jarFileName = jarFile.toString();
- ClassLoaderStack.addJarFile(jarFileName, className);
- LOG.debug("Successfully pushed classloader for jar: " + jarFileName);
- return;
- }
- }
-
- throw new IOException("Could not load shim jar for pattern: "
- + jarPattern);
- }
-
- private ShimLoader() {
- // prevent instantiation
- }
-
- /**
- * Given the name of a class, try to load the shim jars and return the Class
- * object referring to that class.
- * @param className a class to load out of the shim jar
- * @return the class object loaded from the shim jar for the given class.
- */
- public static Class getShimClass(String className)
- throws ClassNotFoundException {
- getHadoopShim(); // Make sure shims are loaded.
- return (Class) Class.forName(className,
- true, Thread.currentThread().getContextClassLoader());
- }
-}
diff --git a/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java b/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java
index 72f31191..5bc54be7 100644
--- a/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java
@@ -41,7 +41,6 @@
import com.cloudera.sqoop.lib.DelimiterSet;
import com.cloudera.sqoop.manager.ConnManager;
import com.cloudera.sqoop.metastore.JobData;
-import com.cloudera.sqoop.shims.ShimLoader;
/**
* Layer on top of SqoopTool that provides some basic common code
@@ -77,8 +76,8 @@ public abstract class BaseSqoopTool extends SqoopTool {
public static final String HIVE_HOME_ARG = "hive-home";
public static final String WAREHOUSE_DIR_ARG = "warehouse-dir";
public static final String TARGET_DIR_ARG = "target-dir";
- public static final String APPEND_ARG = "append";
-
+ public static final String APPEND_ARG = "append";
+
public static final String FMT_SEQUENCEFILE_ARG = "as-sequencefile";
public static final String FMT_TEXTFILE_ARG = "as-textfile";
public static final String HIVE_IMPORT_ARG = "hive-import";
@@ -170,10 +169,6 @@ protected void setManager(ConnManager mgr) {
* @return true on success, false on failure.
*/
protected boolean init(SqoopOptions sqoopOpts) {
-
- // Make sure shim jar is classloaded early.
- ShimLoader.getHadoopShim(sqoopOpts.getConf());
-
// Get the connection to the database.
try {
JobData data = new JobData(sqoopOpts, this);
@@ -227,7 +222,7 @@ protected boolean hasUnrecognizedArgs(String [] argv, int offset, int len) {
LOG.error("Error parsing arguments for " + getToolName() + ":");
printedBanner = true;
}
- LOG.error("Unrecognized argument: " + argv[i]);
+ LOG.error("Unrecognized argument: " + argv[i]);
unrecognized = true;
}
}
@@ -537,7 +532,7 @@ protected RelatedOptions getHBaseOptions() {
}
-
+
/**
* Apply common command-line to the state.
*/
@@ -650,7 +645,7 @@ protected void applyOutputFormatOptions(CommandLine in, SqoopOptions out)
in.getOptionValue(ESCAPED_BY_ARG)));
out.setExplicitDelims(true);
}
-
+
if (in.hasOption(MYSQL_DELIMITERS_ARG)) {
out.setOutputEncloseRequired(false);
out.setFieldsTerminatedBy(',');
diff --git a/src/java/com/cloudera/sqoop/tool/SqoopTool.java b/src/java/com/cloudera/sqoop/tool/SqoopTool.java
index 751d2bf9..d8452780 100644
--- a/src/java/com/cloudera/sqoop/tool/SqoopTool.java
+++ b/src/java/com/cloudera/sqoop/tool/SqoopTool.java
@@ -36,7 +36,7 @@
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
import com.cloudera.sqoop.cli.SqoopParser;
import com.cloudera.sqoop.cli.ToolOptions;
-import com.cloudera.sqoop.shims.ShimLoader;
+import com.cloudera.sqoop.config.ConfigurationHelper;
/**
* Base class for Sqoop subprograms (e.g., SqoopImport, SqoopExport, etc.)
@@ -158,7 +158,7 @@ protected void setToolName(String name) {
* configureOptions()/applyOptions().
* @return an integer return code for external programs to consume. 0
* represents success; nonzero means failure.
- */
+ */
public abstract int run(SqoopOptions options);
/**
@@ -246,7 +246,7 @@ public SqoopOptions parseArguments(String [] args,
String [] toolArgs = args; // args after generic parser is done.
if (useGenericOptions) {
try {
- toolArgs = ShimLoader.getHadoopShim().parseGenericOptions(
+ toolArgs = ConfigurationHelper.parseGenericOptions(
out.getConf(), args);
} catch (IOException ioe) {
ParseException pe = new ParseException(
diff --git a/src/java/com/cloudera/sqoop/util/Jars.java b/src/java/com/cloudera/sqoop/util/Jars.java
index 8c3d62b4..24aefbbc 100644
--- a/src/java/com/cloudera/sqoop/util/Jars.java
+++ b/src/java/com/cloudera/sqoop/util/Jars.java
@@ -27,7 +27,6 @@
import org.apache.commons.logging.LogFactory;
import com.cloudera.sqoop.manager.ConnManager;
-import com.cloudera.sqoop.shims.HadoopShim;
/**
* Utility class; returns the locations of various jars.
@@ -47,17 +46,6 @@ public static String getSqoopJarPath() {
return getJarPathForClass(Jars.class);
}
- /**
- * @return the path to the currently-loaded shim jar.
- */
- public static String getShimJarPath() {
- HadoopShim h = HadoopShim.get();
- if (null == h) {
- return null;
- }
- return getJarPathForClass(h.getClass());
- }
-
/**
* Return the jar file path that contains a particular class.
* Method mostly cloned from o.a.h.mapred.JobConf.findContainingJar().
diff --git a/src/java/com/cloudera/sqoop/util/TaskId.java b/src/java/com/cloudera/sqoop/util/TaskId.java
index e9eb4dfc..baabb1e7 100644
--- a/src/java/com/cloudera/sqoop/util/TaskId.java
+++ b/src/java/com/cloudera/sqoop/util/TaskId.java
@@ -23,7 +23,7 @@
import org.apache.hadoop.conf.Configuration;
-import com.cloudera.sqoop.shims.HadoopShim;
+import com.cloudera.sqoop.config.ConfigurationConstants;
/**
* Utility class; returns task attempt Id of the current job
@@ -52,7 +52,8 @@ public static String get(Configuration conf, String defaultVal) {
* task attempt can be stored.
*/
public static File getLocalWorkPath(Configuration conf) throws IOException {
- String tmpDir = conf.get(HadoopShim.get().getJobLocalDirProperty(),
+ String tmpDir = conf.get(
+ ConfigurationConstants.PROP_JOB_LOCAL_DIRECTORY,
"/tmp/");
// Create a local subdir specific to this task attempt.
diff --git a/src/scripts/hudson/run-code-quality.sh b/src/scripts/hudson/run-code-quality.sh
index 58281bc6..bc77e8c0 100755
--- a/src/scripts/hudson/run-code-quality.sh
+++ b/src/scripts/hudson/run-code-quality.sh
@@ -39,7 +39,7 @@ fi
# Run main compilation step.
-${ANT} clean jar-all-shims findbugs javadoc cobertura checkstyle \
+${ANT} clean jar-all findbugs javadoc cobertura checkstyle \
-Divy.home=$IVY_HOME -Dhadoop.dist=${COMPILE_HADOOP_DIST} \
-Dcobertura.home=${COBERTURA_HOME} -Dcobertura.format=xml \
-Dfindbugs.home=${FINDBUGS_HOME} \
diff --git a/src/shims/apache/com/cloudera/sqoop/shims/Apache22HadoopShim.java b/src/shims/apache/com/cloudera/sqoop/shims/Apache22HadoopShim.java
deleted file mode 100644
index 6f338eb3..00000000
--- a/src/shims/apache/com/cloudera/sqoop/shims/Apache22HadoopShim.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.sqoop.shims;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.MapContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.TaskCounter;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.task.MapContextImpl;
-import org.apache.hadoop.mrunit.mapreduce.mock.MockReporter;
-
-/**
- * Hadoop Shim for Apache 0.22.
- */
-public class Apache22HadoopShim extends CommonHadoopShim {
- @Override
- public long getNumMapOutputRecords(Job job)
- throws IOException, InterruptedException {
- return job.getCounters().findCounter(
- TaskCounter.MAP_OUTPUT_RECORDS).getValue();
- }
-
- @Override
- public long getNumMapInputRecords(Job job)
- throws IOException, InterruptedException {
- return job.getCounters().findCounter(
- TaskCounter.MAP_INPUT_RECORDS).getValue();
- }
-
- @Override
- public String getTaskIdProperty() {
- return JobContext.TASK_ID;
- }
-
- @Override
- public String getJobLocalDirProperty() {
- return JobContext.JOB_LOCAL_DIR;
- }
-
- @Override
- public void setJobNumMaps(Job job, int numMapTasks) {
- job.getConfiguration().setInt(JobContext.NUM_MAPS, numMapTasks);
- }
-
- @Override
- public int getJobNumMaps(JobContext job) {
- return job.getConfiguration().getInt(JobContext.NUM_MAPS, 1);
- }
-
- @Override
- public int getConfNumMaps(Configuration conf) {
- return conf.getInt(JobContext.NUM_MAPS, 1);
- }
-
- @Override
- public void setJobMapSpeculativeExecution(Job job, boolean isEnabled) {
- job.setMapSpeculativeExecution(isEnabled);
- }
-
- @Override
- public void setJobReduceSpeculativeExecution(Job job, boolean isEnabled) {
- job.setReduceSpeculativeExecution(isEnabled);
- }
-
- @Override
- public void setJobtrackerAddr(Configuration conf, String addr) {
- conf.set(JTConfig.JT_IPC_ADDRESS, "local");
- }
-
- private static class MockMapContextWithCommitter
- extends MapContextImpl {
- private Configuration conf;
- private Path path;
-
- public MockMapContextWithCommitter(Configuration c, Path p) {
- super(c, new TaskAttemptID("jt", 0, TaskType.MAP, 0, 0),
- null, null, null, new MockReporter(new Counters()), null);
-
- this.conf = c;
- this.path = p;
- }
-
- @Override
- public InputSplit getInputSplit() {
- return new FileSplit(new Path(path, "inputFile"), 0, 0, new String[0]);
- }
-
- @Override
- public Configuration getConfiguration() {
- return conf;
- }
-
- @Override
- public OutputCommitter getOutputCommitter() {
- try {
- return new FileOutputCommitter(path, this);
- } catch (IOException ioe) {
- return null;
- }
- }
- }
-
- @Override
- public MapContext getMapContextForIOPath(Configuration conf, Path p) {
- return new MockMapContextWithCommitter(conf, p);
- }
-}
diff --git a/src/shims/common/com/cloudera/sqoop/shims/CommonHadoopShim.java b/src/shims/common/com/cloudera/sqoop/shims/CommonHadoopShim.java
deleted file mode 100644
index c5caad9b..00000000
--- a/src/shims/common/com/cloudera/sqoop/shims/CommonHadoopShim.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.shims;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
-import org.apache.hadoop.util.GenericOptionsParser;
-
-/**
- * Contains code which belongs in all Hadoop shims which is syntactically
- * identical in both, but needs to be recompiled against multiple different
- * Hadoop versions (e.g., references to 'final static String' fields).
- */
-public abstract class CommonHadoopShim extends HadoopShim {
-
- @Override
- public String [] parseGenericOptions(Configuration conf, String [] args)
- throws IOException {
- // This needs to be shimmed because in Apache Hadoop this can throw
- // an IOException, but it does not do so in CDH. We just mandate in
- // this method that an IOException is possible.
- GenericOptionsParser genericParser = new GenericOptionsParser(
- conf, args);
- return genericParser.getRemainingArgs();
- }
-
- @Override
- public String getDbInputClassProperty() {
- return DBConfiguration.INPUT_CLASS_PROPERTY;
- }
-
- @Override
- public String getDbUsernameProperty() {
- return DBConfiguration.USERNAME_PROPERTY;
- }
-
- @Override
- public String getDbPasswordProperty() {
- return DBConfiguration.PASSWORD_PROPERTY;
- }
-
- @Override
- public String getDbUrlProperty() {
- return DBConfiguration.URL_PROPERTY;
- }
-
- @Override
- public String getDbInputTableNameProperty() {
- return DBConfiguration.INPUT_TABLE_NAME_PROPERTY;
- }
-
- @Override
- public String getDbInputConditionsProperty() {
- return DBConfiguration.INPUT_CONDITIONS_PROPERTY;
- }
-}
-
diff --git a/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java b/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java
index 4346d08e..a7f299c3 100644
--- a/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java
+++ b/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java
@@ -18,7 +18,9 @@
package com.cloudera.sqoop.lib;
-import java.io.*;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -29,7 +31,8 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.MapContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import com.cloudera.sqoop.shims.HadoopShim;
+
+import com.cloudera.sqoop.testutil.MockObjectFactory;
import com.cloudera.sqoop.testutil.MockResultSet;
/**
@@ -61,7 +64,7 @@ public void setUp() throws IOException, InterruptedException {
* getInputSplit() to determine where to read our source data from--the same
* directory. We are repurposing the same context for both output and input.
*/
- mapContext = HadoopShim.get().getMapContextForIOPath(conf, outDir);
+ mapContext = MockObjectFactory.getMapContextForIOPath(conf, outDir);
loader = new LargeObjectLoader(mapContext.getConfiguration(),
FileOutputFormat.getWorkOutputPath(mapContext));
}
diff --git a/src/test/com/cloudera/sqoop/orm/TestParseMethods.java b/src/test/com/cloudera/sqoop/orm/TestParseMethods.java
index 58df3e60..acef3880 100644
--- a/src/test/com/cloudera/sqoop/orm/TestParseMethods.java
+++ b/src/test/com/cloudera/sqoop/orm/TestParseMethods.java
@@ -32,7 +32,7 @@
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.shims.HadoopShim;
+import com.cloudera.sqoop.config.ConfigurationHelper;
import com.cloudera.sqoop.testutil.CommonArgs;
import com.cloudera.sqoop.testutil.HsqldbTestServer;
import com.cloudera.sqoop.testutil.ImportJobTestCase;
@@ -50,7 +50,7 @@ public class TestParseMethods extends ImportJobTestCase {
* Create the argv to pass to Sqoop.
* @return the argv as an array of strings.
*/
- private String [] getArgv(boolean includeHadoopFlags, String fieldTerminator,
+ private String [] getArgv(boolean includeHadoopFlags, String fieldTerminator,
String lineTerminator, String encloser, String escape,
boolean encloserRequired) {
@@ -118,7 +118,7 @@ public void runParseTest(String fieldTerminator, String lineTerminator,
job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
// use local mode in the same JVM.
- HadoopShim.get().setJobtrackerAddr(job, "local");
+ ConfigurationHelper.setJobtrackerAddr(job, "local");
job.set("fs.default.name", "file:///");
String warehouseDir = getWarehouseDir();
@@ -203,7 +203,7 @@ public void testNumericTypes() throws IOException {
"92104916282869291837672829102857271948687.287475322",
"true",
};
-
+
createTableWithColTypes(types, vals);
runParseTest(",", "\\n", "\\\'", "\\", false);
}
diff --git a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
index 72104fe6..f6d24748 100644
--- a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
+++ b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
@@ -39,7 +39,6 @@
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.manager.ConnManager;
import com.cloudera.sqoop.metastore.JobData;
-import com.cloudera.sqoop.shims.ShimLoader;
import com.cloudera.sqoop.tool.ImportTool;
import junit.framework.TestCase;
@@ -168,7 +167,6 @@ protected SqoopOptions getSqoopOptions(Configuration conf) {
@Before
public void setUp() {
- ShimLoader.getHadoopShim();
incrementTableNum();
if (!isLog4jConfigured) {
diff --git a/src/shims/cloudera/com/cloudera/sqoop/shims/CDH3Shim.java b/src/test/com/cloudera/sqoop/testutil/MockObjectFactory.java
similarity index 53%
rename from src/shims/cloudera/com/cloudera/sqoop/shims/CDH3Shim.java
rename to src/test/com/cloudera/sqoop/testutil/MockObjectFactory.java
index 12122be1..e8bca5f6 100644
--- a/src/shims/cloudera/com/cloudera/sqoop/shims/CDH3Shim.java
+++ b/src/test/com/cloudera/sqoop/testutil/MockObjectFactory.java
@@ -15,7 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.cloudera.sqoop.shims;
+
+package com.cloudera.sqoop.testutil;
import java.io.IOException;
@@ -23,8 +24,6 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.MapContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -33,65 +32,17 @@
import org.apache.hadoop.mrunit.mapreduce.mock.MockReporter;
/**
- * Hadoop Shim for CDH3 (based on 0.20.2).
+ * Allows the creation of various mock objects for testing purposes.
*/
-public class CDH3Shim extends CommonHadoopShim {
- @Override
- public long getNumMapOutputRecords(Job job)
- throws IOException, InterruptedException {
- return job.getCounters().findCounter(
- "org.apache.hadoop.mapred.Task$Counter",
- "MAP_OUTPUT_RECORDS").getValue();
- }
+public final class MockObjectFactory {
- @Override
- public long getNumMapInputRecords(Job job)
- throws IOException, InterruptedException {
- return job.getCounters().findCounter(
- "org.apache.hadoop.mapred.Task$Counter",
- "MAP_INPUT_RECORDS").getValue();
- }
-
- @Override
- public String getTaskIdProperty() {
- return "mapred.task.id";
- }
-
- @Override
- public String getJobLocalDirProperty() {
- return "job.local.dir";
- }
-
- @Override
- public void setJobNumMaps(Job job, int numMapTasks) {
- job.getConfiguration().setInt("mapred.map.tasks", numMapTasks);
- }
-
- @Override
- public int getJobNumMaps(JobContext job) {
- return job.getConfiguration().getInt("mapred.map.tasks", 1);
- }
-
- @Override
- public int getConfNumMaps(Configuration conf) {
- return conf.getInt("mapred.map.tasks", 1);
- }
-
- @Override
- public void setJobMapSpeculativeExecution(Job job, boolean isEnabled) {
- job.getConfiguration().setBoolean(
- "mapred.map.tasks.speculative.execution", isEnabled);
- }
-
- @Override
- public void setJobReduceSpeculativeExecution(Job job, boolean isEnabled) {
- job.getConfiguration().setBoolean(
- "mapred.reduce.tasks.speculative.execution", isEnabled);
- }
-
- @Override
- public void setJobtrackerAddr(Configuration conf, String addr) {
- conf.set("mapred.job.tracker", addr);
+ /**
+ * Returns a mock MapContext that has both an OutputCommitter and an
+ * InputSplit wired to the specified path.
+ * Used for testing LargeObjectLoader.
+ */
+ public static MapContext getMapContextForIOPath(Configuration conf, Path p) {
+ return new MockMapContextWithCommitter(conf, p);
}
private static class MockMapContextWithCommitter
@@ -101,7 +52,7 @@ private static class MockMapContextWithCommitter
public MockMapContextWithCommitter(Configuration c, Path p) {
super(c, new TaskAttemptID("jt", 0, true, 0, 0),
- null, null, null, new MockReporter(new Counters()), null);
+ null, null, null, new MockReporter(new Counters()), null);
this.path = p;
this.conf = c;
@@ -127,8 +78,7 @@ public Configuration getConfiguration() {
}
}
- @Override
- public MapContext getMapContextForIOPath(Configuration conf, Path p) {
- return new MockMapContextWithCommitter(conf, p);
+ private MockObjectFactory() {
+ // Disable explicity object creation
}
}