github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-02 15:12:33 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1261: CompilationManager should add Hadoop 2.x libraries to the classpath under Hadoop 2.x

Browse Source 
(Venkat Ranganathan via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2013-12-15 13:57:15 -08:00
parent 6523ebb176
commit 28f415ff2f
1 changed files with 23 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/java/org/apache/sqoop/orm/CompilationManager.java
View File

@ -24,6 +24,7 @@
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.jar.JarOutputStream; import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
@ -77,7 +78,7 @@ public void addSourceFile(String sourceName) {
* If that doesn't work, check our classpath. * If that doesn't work, check our classpath.
* @return the filename of the hadoop-*-core.jar file. * @return the filename of the hadoop-*-core.jar file.
*/ */
private String findHadoopCoreJar() { private String findHadoopJars() {
String hadoopMapRedHome = options.getHadoopMapRedHome(); String hadoopMapRedHome = options.getHadoopMapRedHome();
if (null == hadoopMapRedHome) { if (null == hadoopMapRedHome) {
@ -91,22 +92,30 @@ private String findHadoopCoreJar() {
File hadoopMapRedHomeFile = new File(hadoopMapRedHome); File hadoopMapRedHomeFile = new File(hadoopMapRedHome);
LOG.info("HADOOP_MAPRED_HOME is " + hadoopMapRedHomeFile.getAbsolutePath()); LOG.info("HADOOP_MAPRED_HOME is " + hadoopMapRedHomeFile.getAbsolutePath());
File [] entries = hadoopMapRedHomeFile.listFiles();
Iterator<File> filesIterator = FileUtils.iterateFiles(hadoopMapRedHomeFile,
new String[] { "jar" }, true);
StringBuilder sb = new StringBuilder();
if (null == entries) { while (filesIterator.hasNext()) {
File file = filesIterator.next();
String name = file.getName();
if (name.startsWith("hadoop-common")
|| name.startsWith("hadoop-mapreduce-client-core")
|| name.startsWith("hadoop-core")) {
sb.append(file.getAbsolutePath());
sb.append(File.pathSeparator);
}
}
if (sb.length() < 1) {
LOG.warn("HADOOP_MAPRED_HOME appears empty or missing"); LOG.warn("HADOOP_MAPRED_HOME appears empty or missing");
return Jars.getJarPathForClass(JobConf.class); return Jars.getJarPathForClass(JobConf.class);
} }
for (File f : entries) { String s = sb.substring(0, sb.length() - 1);
if (f.getName().startsWith("hadoop-") LOG.debug("Returning jar file path " + s);
&& f.getName().endsWith("-core.jar")) { return s;
LOG.info("Found hadoop core jar at: " + f.getAbsolutePath());
return f.getAbsolutePath();
}
}
return Jars.getJarPathForClass(JobConf.class);
} }
/** /**
@ -134,7 +143,7 @@ public void compile() throws IOException {
} }
// find hadoop-*-core.jar for classpath. // find hadoop-*-core.jar for classpath.
String coreJar = findHadoopCoreJar(); String coreJar = findHadoopJars();
if (null == coreJar) { if (null == coreJar) {
// Couldn't find a core jar to insert into the CP for compilation. If, // Couldn't find a core jar to insert into the CP for compilation. If,
// however, we're running this from a unit test, then the path to the // however, we're running this from a unit test, then the path to the
@ -159,6 +168,7 @@ public void compile() throws IOException {
} }
String curClasspath = System.getProperty("java.class.path"); String curClasspath = System.getProperty("java.class.path");
LOG.debug("Current sqoop classpath = " + curClasspath);
args.add("-sourcepath"); args.add("-sourcepath");
args.add(jarOutDir); args.add(jarOutDir);