Sqoop can compile against locally-installed Hadoop

From: Aaron Kimball <aaron@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149929 13f79535-47bb-0310-9956-ffa450edef68

COMPILING.txt

@@ -105,6 +105,26 @@ ant jar -Dhadoop.dist=cloudera
 To switch between builds, you will need to clear Ivy's dependency
 cache: +ant veryclean+
 
+=== Using a Local Hadoop Installation
+
+Sqoop can be compiled against a locally-installed version of Sqoop,
+bypassing the maven repository. To do this you will need to set 
+three properties:
+
+- hadoop.dist should be set to "local"
+- hadoop.shim should be set to "cloudera" or "apache", to tell Sqoop whether
+  to build the Cloudera or Apache-specific shim jar
+- hadoop.home should be set to the path where Hadoop is installed.
+
+For example, the following will compile Sqoop against a locally-installed
+version of CDH.
+
+++++
+ant jar jar-one-shim -Dhadoop.dist=local -Dhadoop.shim=cloudera \
+    -Dhadoop.home=/usr/lib/hadoop
+++++
+
+
 == Code Quality Analysis
 
 We have three tools which can be used to analyze Sqoop's code quality.

build.xml

@@ -142,19 +142,6 @@
   <property name="ivy.artifact.retrieve.pattern"
     			value="${name}/[conf]/[artifact]-[revision].[ext]"/>
 
-  <!-- The classpath for compiling and running Sqoop -->
-  <path id="compile.classpath">
-    <pathelement location="${build.classes}"/>
-    <path refid="lib.path"/>
-    <path refid="${name}.hadoop.classpath"/>
-  </path>
-
-  <path id="cobertura.classpath">
-    <fileset dir="${cobertura.home}">
-      <include name="**/*.jar" />
-    </fileset>
-  </path>
-
   <!-- load ant-contrib tasks to get the "if" task. -->
   <taskdef resource="net/sf/antcontrib/antcontrib.properties">
     <classpath>
@@ -162,6 +149,41 @@
     </classpath>
   </taskdef>
 
+  <!-- The classpath for compiling and running Sqoop -->
+  <if>
+    <isset property="hadoop.home" />
+    <then>
+      <path id="compile.classpath">
+        <pathelement location="${build.classes}"/>
+        <path refid="lib.path"/>
+        <fileset dir="${hadoop.home}">
+          <include name="hadoop-core-*.jar" />
+          <include name="hadoop-*-core.jar" />
+          <include name="hadoop-common-*.jar" />
+          <include name="hadoop-mapred-*.jar" />
+          <include name="hadoop-hdfs-*.jar" />
+        </fileset>
+        <fileset dir="${hadoop.home}/lib">
+          <include name="*.jar" />
+        </fileset>
+        <path refid="${name}.hadoop.classpath"/>
+      </path>
+    </then>
+    <else>
+      <path id="compile.classpath">
+        <pathelement location="${build.classes}"/>
+        <path refid="lib.path"/>
+        <path refid="${name}.hadoop.classpath"/>
+      </path>
+    </else>
+  </if>
+
+  <path id="cobertura.classpath">
+    <fileset dir="${cobertura.home}">
+      <include name="**/*.jar" />
+    </fileset>
+  </path>
+
   <!-- "init" target used for setup purposes. --> 
   <target name="init">
     <!-- The hadoop.dist property determines which version of Hadoop to
@@ -189,6 +211,21 @@
           <echo message="Hadoop distribution: cloudera -> cloudera (CDH3)" />
         </then>
       </elseif>
+      <elseif>
+        <equals arg1="${hadoop.dist}" arg2="local" />
+        <then>
+          <echo message="Hadoop distribution set to local installation" />
+          <property name="hadoop.is.local" value="true" />
+          <fail unless="hadoop.shim">
+            You must explicitly set ${hadoop.shim} to 'apache' or 'cloudera'
+            to use a local distribution.
+          </fail>
+          <fail unless="hadoop.home">
+            You must set ${hadoop.home} to point to your local
+            Hadoop installation.
+          </fail>
+        </then>
+      </elseif>
       <else>
         <fail message="Invalid value for hadoop.dist: ${hadoop.dist}"/>
       </else>
@@ -933,25 +970,67 @@
   </target>
 
   <!-- retrieve ivy-managed artifacts from the Hadoop distribution -->
-  <target name="ivy-resolve-hadoop" depends="ivy-init">
+  <target name="ivy-resolve-hadoop" depends="ivy-init"
+      unless="hadoop.is.local">
     <ivy:resolve settingsRef="${name}.ivy.settings" conf="${hadoop.dist}" />
   </target>
-  <target name="ivy-retrieve-hadoop" depends="ivy-resolve-hadoop">
+  <target name="ivy-retrieve-hadoop" depends="ivy-init,ivy-resolve-hadoop">
-    <ivy:retrieve settingsRef="${name}.ivy.settings" 
+    <if>
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}" sync="true" />
+      <equals arg1="${hadoop.dist}" arg2="local" />
-    <ivy:cachepath pathid="${name}.hadoop.classpath" conf="${hadoop.dist}" />
+      <then>
+        <!-- Use a local Hadoop distribution. Just retrieve the basic
+             'common' configuration, and add the Hadoop jars from
+             the local Hadoop install.
+             We can't use a subant here, because the refs wouldn't be
+             preserved when we return to executing this target. 
+        -->
+        <ivy:resolve settingsRef="${name}.ivy.settings" conf="common" />
+        <ivy:retrieve settingsRef="${name}.ivy.settings" 
+            pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+            sync="true" />
+        <ivy:cachepath pathid="${name}.hadoop.classpath" conf="common" />
+      </then>
+      <else>
+        <!-- retrieve hadoop refs normally. -->
+        <ivy:retrieve settingsRef="${name}.ivy.settings" 
+          pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+          sync="true" />
+        <ivy:cachepath pathid="${name}.hadoop.classpath" conf="${hadoop.dist}" />
+      </else>
+    </if>
   </target>
 
   <!-- retrieve ivy-managed test artifacts from the Hadoop distribution -->
-  <target name="ivy-resolve-hadoop-test" depends="ivy-init">
+  <target name="ivy-resolve-hadoop-test" depends="ivy-init"
+      unless="hadoop.is.local">
     <ivy:resolve settingsRef="${name}.ivy.settings" conf="${hadoop.dist}test" />
   </target>
-  <target name="ivy-retrieve-hadoop-test" depends="ivy-resolve-hadoop-test">
+  <target name="ivy-retrieve-hadoop-test"
-    <ivy:retrieve settingsRef="${name}.ivy.settings" 
+      depends="ivy-init,ivy-resolve-hadoop-test">
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+    <if>
-      sync="true" />
+      <equals arg1="${hadoop.dist}" arg2="local" />
-    <ivy:cachepath pathid="${name}.hadooptest.classpath"
+      <then>
-      conf="${hadoop.dist}test" />
+        <!-- Use a local Hadoop distribution. Just retrieve the basic
+             'test' configuration, and add the Hadoop jars from
+             the local Hadoop install.
+             We can't use a subant here, because the refs wouldn't be
+             preserved when we return to executing this target. 
+        -->
+        <ivy:resolve settingsRef="${name}.ivy.settings" conf="test" />
+        <ivy:retrieve settingsRef="${name}.ivy.settings" 
+            pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+            sync="true" />
+        <ivy:cachepath pathid="${name}.hadooptest.classpath" conf="test" />
+      </then>
+      <else>
+        <!-- retrieve hadoop refs normally. -->
+        <ivy:retrieve settingsRef="${name}.ivy.settings" 
+          pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+          sync="true" />
+        <ivy:cachepath pathid="${name}.hadooptest.classpath"
+          conf="${hadoop.dist}test" />
+      </else>
+    </if>
   </target>
 
   <!-- retrieve ivy-managed artifacts for checkstyle -->