SQOOP-108. Automatically obtain HBase and ZK deps.

Users currently need to specify hbase.home and zk.home in build.properties.
This change helps automatically resolve these dependencies by downloading
release tarballs. Would be best to do this via SQOOP-101 but the hbase and
zk maven layout currently has some issues that are painful to workaround
in ivy.

Reason: Improvement
Author: Eli Collins via Arvind Prabhakar

From: Arvind Prabhakar <arvind@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149986 13f79535-47bb-0310-9956-ffa450edef68

COMPILING.txt

@@ -43,32 +43,6 @@ Furthermore, Sqoop's build can be instrumented with the following:
 * cobertura (1.9.4.1) for code coverage
 * checkstyle (5.x) for code style checks
 
-Finally, while most of Sqoop's compilation dependencies are in the lib/
-directory, or retrieved via Ivy, the following dependencies must be installed
-on your system:
-
-* HBase
-* Zookeeper
-
-Both of these must be from the most recent version of CDH. These can be
-installed on linux machines via yum or apt-get; any users may also use the
-tarballs available at http://archive.cloudera.com/cdh/3/.
-
-== Configuring your build
-
-The location of HBase and Zookeeper must be specified to the Sqoop build via
-ant properties. All compilation / testing requires the +hbase.home+ and
-+zookeeper.home+ properties to be set. This can be done on the command line
-(e.g.: +ant jar -Dhbase.home=/path/to/hbase -Dzookeeper.home=/path/to/zk+), or
-in a properties file.
-
-Sqoop will automatically load properties out of a file named
-'build.properties' in your Sqoop source root, or your home directory. You
-should create such a file and include the lines:
-
-hbase.home=/path/to/hbase
-zookeeper.home=/path/to/zk
-
 == The Basics
 
 Sqoop is compiled with ant. Type +ant -p+ to see the list of available targets.

build.xml

@@ -189,13 +189,42 @@
     </classpath>
   </taskdef>
 
+  <!-- Download/untar HBase and ZooKeeper -->
+  <property name="cdh.tar.dir" value="http://archive.cloudera.com/cdh/3"/>
+  <property name="hbase.tar" value="hbase-${hbase.version}.tar.gz"/>
+  <property name="hbase.tar.url" value="${cdh.tar.dir}/${hbase.tar}"/>
+  <property name="hbase.dir" location="${build.dir}/hbase-${hbase.version}" />
+  <property name="zk.tar" value="zookeeper-${zookeeper.version}.tar.gz"/>
+  <property name="zk.tar.url" value="${cdh.tar.dir}/${zk.tar}"/>
+  <property name="zk.dir" location="${build.dir}/zookeeper-${zookeeper.version}" />
+
+ <target name="download-hbase-zk" unless="offline">
+   <get dest="${build.dir}/${hbase.tar}" src="${hbase.tar.url}"
+        usetimestamp="true" verbose="true"/>
+   <get dest="${build.dir}/${zk.tar}" src="${zk.tar.url}" usetimestamp="true"
+        verbose="true"/>
+   <condition property="hbase-zk-already-untared">
+      <and>
+         <available file="${hbase.dir}" type="dir"/>
+         <available file="${zk.dir}" type="dir"/>
+      </and>
+   </condition>
+ </target>
+
+ <target name="untar-hbase-zk" unless="hbase-zk-already-untared" 
+	 depends="download-hbase-zk">
+   <untar src="${build.dir}/${hbase.tar}" dest="${build.dir}" 
+	  compression="gzip"/>
+   <untar src="${build.dir}/${zk.tar}" dest="${build.dir}" compression="gzip"/>
+ </target>
+
   <!-- manually-specified HBase classpath til it works with Ivy. -->
   <path id="hbase.classpath">
-    <fileset dir="${hbase.home}">
+    <fileset dir="${hbase.dir}">
       <include name="*.jar" />
       <include name="lib/*.jar" />
     </fileset>
-    <fileset dir="${zookeeper.home}">
+    <fileset dir="${zk.dir}">
       <include name="*.jar" />
       <include name="lib/*.jar" />
     </fileset>
@@ -350,7 +379,7 @@
   
   <!-- Compile core classes for the project -->
   <target name="compile"
-      depends="init, gen-version, ivy-retrieve-hadoop"
+      depends="init, gen-version, ivy-retrieve-hadoop, untar-hbase-zk"
       description="Compile core classes for the project">
     <!-- don't use an out-of-date instrumented build. -->
     <delete dir="${cobertura.class.dir}" />

ivy/libraries.properties

@@ -37,7 +37,8 @@ hadoop-mapred.apache21.version=0.21.0-SNAPSHOT
 # Cloudera Distribution dependency version
 hadoop-core.cloudera.version=0.20.2-314
 
-hbase.version=0.89.20100621
+hbase.version=0.89.20100621+17
+zookeeper.version=3.3.1+7
 
 hsqldb.version=1.8.0.10
 

src/scripts/hudson/run-code-quality.sh

@@ -43,7 +43,6 @@ ${ANT} clean jar-all-shims findbugs javadoc cobertura checkstyle \
     -Divy.home=$IVY_HOME -Dhadoop.dist=${COMPILE_HADOOP_DIST} \
     -Dcobertura.home=${COBERTURA_HOME} -Dcobertura.format=xml \
     -Dfindbugs.home=${FINDBUGS_HOME} \
-    -Dhbase.home=${HBASE_HOME} -Dzookeeper.home=${ZOOKEEPER_HOME} \
     -Dtest.junit.output.format=xml ${ANT_ARGUMENTS}
 
 if [ "$?" != "0" ]; then
@@ -57,7 +56,6 @@ ${ANT} cobertura \
     -Dhadoop.dist=${COMPILE_HADOOP_DIST} \
     -Dcobertura.home=${COBERTURA_HOME} -Dcobertura.format=xml \
     -Dsqoop.thirdparty.lib.dir=${THIRDPARTY_LIBS} \
-    -Dhbase.home=${HBASE_HOME} -Dzookeeper.home=${ZOOKEEPER_HOME} \
     -Dtestcase=ThirdPartyTests ${ANT_ARGUMENTS}
 
 if [ "$?" != "0" ]; then

src/scripts/hudson/run-tests.sh

@@ -30,7 +30,6 @@ source ${bin}/test-config.sh
 # Run compilation step.
 
 ${ANT} clean jar -Divy.home=$IVY_HOME -Dhadoop.dist=${COMPILE_HADOOP_DIST} \
-    -Dhbase.home=${HBASE_HOME} -Dzookeeper.home=${ZOOKEEPER_HOME} \
     ${ANT_ARGUMENTS}
 if [ "$?" != "0" ]; then
   echo "Error during compilation phase. Aborting!"
@@ -42,7 +41,6 @@ testfailed=0
 # Run basic unit tests.
 
 ${ANT} clean-cache test -Divy.home=$IVY_HOME -Dtest.junit.output.format=xml \
-    -Dhbase.home=${HBASE_HOME} -Dzookeeper.home=${ZOOKEEPER_HOME} \
     -Dhadoop.dist=${TEST_HADOOP_DIST} ${ANT_ARGUMENTS}
 if [ "$?" != "0" ]; then
   testfailed=1
@@ -56,7 +54,6 @@ fi
 
 ${ANT} test -Dthirdparty=true -Dsqoop.thirdparty.lib.dir=${THIRDPARTY_LIBS} \
     -Dtest.junit.output.format=xml -Divy.home=$IVY_HOME \
-    -Dhbase.home=${HBASE_HOME} -Dzookeeper.home=${ZOOKEEPER_HOME} \
     -Dhadoop.dist=${TEST_HADOOP_DIST} ${ANT_ARGUMENTS}
 if [ "$?" != "0" ]; then
   testfailed=1