Add Hudson test scripts.

Add Hadoop 0.21 build support. -Dhadoop.dist now accepts 'apache21'.

From: Aaron Kimball <aaron@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149905 13f79535-47bb-0310-9956-ffa450edef68

COMPILING.txt

@@ -75,18 +75,18 @@ can be used as the underlying Hadoop implementation.
 
 By default, Sqoop will compile against the latest snapshot from Apache
 (retrieved through maven). You can specify the Hadoop distribution to
-retrieve with the hadoop.dist property. Valid values are "apache" or
-"cloudera":
+retrieve with the hadoop.dist property. Valid values are "apache",
+"apache21", or "cloudera":
 
 ++++
-ant jar -Dhadoop.dist=apache
+ant jar -Dhadoop.dist=apache # Uses the nightly build of "trunk"
+ant jar -Dhadoop.dist=apache21
 ant jar -Dhadoop.dist=cloudera
 ++++
 
 To switch between builds, you will need to clear Ivy's dependency
 cache: +ant veryclean+
 
-
 == Code Quality Analysis
 
 We have three tools which can be used to analyze Sqoop's code quality.

build.xml

@@ -65,7 +65,8 @@
   <property name="build.encoding" value="ISO-8859-1"/>
 
   <!-- controlling the Hadoop source -->
-  <!-- valid values for ${hadoop.dist} are 'apache' and 'cloudera' -->
+  <!-- valid values for ${hadoop.dist} are 'apache' (trunk), 'apache21',
+       and 'cloudera' -->
   <property name="hadoop.dist" value="apache" />
 
   <!-- testing with JUnit -->
@@ -118,17 +119,6 @@
   <property name="ivy.artifact.retrieve.pattern"
     			value="${name}/[conf]/[artifact]-[revision].[ext]"/>
 
-  <!-- Path containing third-party libraries deployed directly with Sqoop.
-       This does not include anything that Ivy can retrieve for us.
-    -->
-  <path id="lib.path">
-    <fileset dir="${lib.dir}">
-      <include name="*.jar" />
-      <include name="${hadoop.dist}/*.jar" />
-    </fileset>
-  </path>
-
-
   <!-- The classpath for compiling and running Sqoop -->
   <path id="compile.classpath">
     <pathelement location="${build.classes}"/>
@@ -136,34 +126,83 @@
     <path refid="${name}.hadoop.classpath"/>
   </path>
 
-  <!-- Classpath for unit tests (superset of compile.classpath) -->
-  <path id="test.classpath">
-    <pathelement location="${build.test.classes}" />
-    <path refid="${name}.hadooptest.classpath" />
-    <path refid="compile.classpath" />
-    <pathelement location="${build.shim.classes}/${hadoop.dist}" />
-  </path>
-
   <path id="cobertura.classpath">
     <fileset dir="${cobertura.home}">
       <include name="**/*.jar" />
     </fileset>
   </path>
 
-  <!-- "init" target reserved for future setup purposes. --> 
-  <target name="init" />
+  <!-- load ant-contrib tasks to get the "if" task. -->
+  <taskdef resource="net/sf/antcontrib/antcontrib.properties">
+    <classpath>
+      <pathelement location="${basedir}/lib/ant-contrib-1.0b3.jar"/>
+    </classpath>
+  </taskdef>
 
-  <!-- Compile a shim class so Sqoop can run with the specified hadoop.dist -->
+  <!-- "init" target used for setup purposes. --> 
+  <target name="init">
+    <!-- The hadoop.dist property determines which version of Hadoop to
+         retrieve; this may be "cloudera", "apache", or "apache21". But
+         multiple of these versions can use the same shim. We set the
+         hadoop.shim property here, based on that one.
+    -->
+    <if>
+      <equals arg1="${hadoop.dist}" arg2="apache" />
+      <then>
+        <property name="hadoop.shim" value="apache" />
+        <echo message="Hadoop distribution: apache -> apache (trunk) " />
+      </then>
+      <elseif>
+        <equals arg1="${hadoop.dist}" arg2="apache21" />
+        <then>
+          <property name="hadoop.shim" value="apache" />
+          <echo message="Hadoop distribution: apache21 -> apache (0.21)" />
+        </then>
+      </elseif>
+      <elseif>
+        <equals arg1="${hadoop.dist}" arg2="cloudera" />
+        <then>
+          <property name="hadoop.shim" value="cloudera" />
+          <echo message="Hadoop distribution: cloudera -> cloudera (CDH3)" />
+        </then>
+      </elseif>
+      <else>
+        <fail message="Invalid value for hadoop.dist: ${hadoop.dist}"/>
+      </else>
+    </if>
+
+    <!-- Path containing third-party libraries deployed directly with Sqoop.
+         This does not include anything that Ivy can retrieve for us.
+      -->
+    <path id="lib.path">
+      <fileset dir="${lib.dir}">
+        <include name="*.jar" />
+        <include name="${hadoop.shim}/*.jar" />
+      </fileset>
+    </path>
+
+    <!-- Classpath for unit tests (superset of compile.classpath) -->
+    <path id="test.classpath">
+      <pathelement location="${build.test.classes}" />
+      <path refid="${name}.hadooptest.classpath" />
+      <path refid="compile.classpath" />
+      <pathelement location="${build.shim.classes}/${hadoop.shim}" />
+    </path>
+  </target>
+
+  <!-- Compile a shim class so Sqoop can run with the specified
+       hadoop.dist.
+  -->
   <target name="compile-one-shim"
       depends="init, ivy-retrieve-hadoop">
-    <mkdir dir="${build.shim.classes}/${hadoop.dist}" />
+    <mkdir dir="${build.shim.classes}/${hadoop.shim}" />
     <javac
         encoding="${build.encoding}"
-        destdir="${build.shim.classes}/${hadoop.dist}"
+        destdir="${build.shim.classes}/${hadoop.shim}"
         debug="${javac.debug}"
         deprecation="${javac.deprecation}">
       <classpath refid="compile.classpath"/>
-      <src path="${shim.src.dir}/${hadoop.dist}" />
+      <src path="${shim.src.dir}/${hadoop.shim}" />
       <src path="${shim.src.dir}/common" />
       <include name="**/*.java" />
     </javac>
@@ -171,8 +210,8 @@
 
   <!-- Compile one shim and fetch it into a jar file. -->
   <target name="jar-one-shim" depends="init,compile-one-shim">
-    <jar jarfile="${build.shim.dir}/${name}-${version}-shim-${hadoop.dist}.jar"
-        basedir="${build.shim.classes}/${hadoop.dist}" />
+    <jar jarfile="${build.shim.dir}/${name}-${version}-shim-${hadoop.shim}.jar"
+        basedir="${build.shim.classes}/${hadoop.shim}" />
   </target>
 
   <!-- Create jars for all shims -->

ivy.xml

@@ -35,6 +35,9 @@
     <conf name="apache" visibility="private"
       extends="common,runtime"
       description="artifacts from Apache for compile/test" />
+    <conf name="apache21" visibility="private"
+      extends="common,runtime"
+      description="artifacts from Apache 0.21 for compile/test" />
     <conf name="cloudera" visibility="private"
       extends="common,runtime"
       description="artifacts from Cloudera for compile/test" />
@@ -43,6 +46,9 @@
     <conf name="apachetest" visibility="private"
       extends="test,apache"
       description="artifacts from Apache for testing" />
+    <conf name="apache21test" visibility="private"
+      extends="test,apache21"
+      description="artifacts from Apache 0.21 for testing" />
     <conf name="clouderatest" visibility="private"
       extends="test,cloudera"
       description="artifacts from Cloudera for testing" />
@@ -62,7 +68,7 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
-    <!-- Dependencies for Apache Hadoop -->
+    <!-- Dependencies for Apache Hadoop (trunk) -->
     <dependency org="org.apache.hadoop" name="hadoop-core"
       rev="${hadoop-core.apache.version}" conf="apache->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-core-test"
@@ -76,6 +82,20 @@
     <dependency org="org.apache.hadoop" name="hadoop-mapred-test"
       rev="${hadoop-mapred.apache.version}" conf="apachetest->default"/>
 
+    <!-- Dependencies for Apache Hadoop (0.21) -->
+    <dependency org="org.apache.hadoop" name="hadoop-core"
+      rev="${hadoop-core.apache21.version}" conf="apache21->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-core-test"
+      rev="${hadoop-core.apache21.version}" conf="apache21test->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-hdfs"
+      rev="${hadoop-hdfs.apache21.version}" conf="apache21->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-hdfs-test"
+      rev="${hadoop-hdfs.apache21.version}" conf="apache21test->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-mapred"
+      rev="${hadoop-mapred.apache21.version}" conf="apache21->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-mapred-test"
+      rev="${hadoop-mapred.apache21.version}" conf="apache21test->default"/>
+
     <!-- Dependencies for Cloudera's Distribution for Hadoop -->
     <dependency org="org.apache.hadoop" name="hadoop-core"
       rev="${hadoop-core.cloudera.version}" conf="cloudera->default"/>

ivy/libraries.properties

@@ -27,6 +27,11 @@ hadoop-core.apache.version=0.22.0-SNAPSHOT
 hadoop-hdfs.apache.version=0.22.0-SNAPSHOT
 hadoop-mapred.apache.version=0.22.0-SNAPSHOT
 
+# Apache 0.21 release Hadoop dependency version: use 0.21.
+hadoop-core.apache21.version=0.21.0-SNAPSHOT
+hadoop-hdfs.apache21.version=0.21.0-SNAPSHOT
+hadoop-mapred.apache21.version=0.21.0-SNAPSHOT
+
 # Cloudera Distribution dependency version
 hadoop-core.cloudera.version=0.20.2-CDH3b2-SNAPSHOT
 

src/java/org/apache/hadoop/sqoop/shims/ShimLoader.java

@@ -89,6 +89,12 @@ public abstract class ShimLoader {
     HADOOP_SHIM_MATCHES.add("Unknown");
     HADOOP_SHIM_CLASSES.add("org.apache.hadoop.sqoop.shims.Apache22HadoopShim");
     HADOOP_SHIM_JARS.add("sqoop-.*-apache.jar");
+
+    // Apache 0.21 uses the same shim jars as 0.22
+    HADOOP_SHIM_MATCHES.add("0.21.\\d+(-.*)?");
+    HADOOP_SHIM_CLASSES.add("org.apache.hadoop.sqoop.shims.Apache22HadoopShim");
+    HADOOP_SHIM_JARS.add("sqoop-.*-apache.jar");
+
   }
 
   /**

src/scripts/hudson/run-code-quality.sh

@@ -0,0 +1,65 @@
+#!/bin/bash -x
+#
+# Licensed to Cloudera, Inc. under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Compiles the world and runs code quality/test coverage checks.
+# This script is intended for execution by users who want to thoroughly
+# execute all tests, or automated testing agents such as Hudson.
+
+# Environment:
+# See test-config.sh
+
+bin=`readlink -f $0`
+bin=`dirname ${bin}`
+bin=`cd ${bin} && pwd`
+source ${bin}/test-config.sh
+
+if [ -z "${FINDBUGS_HOME}" ]; then
+  echo "Error: $$FINDBUGS_HOME is not set."
+  exit 1
+fi
+
+if [ -z "${COBERTURA_HOME}" ]; then
+  echo "Error: $$COBERTURA_HOME is not set."
+  exit 1
+fi
+
+# Run main compilation step.
+
+${ANT} clean jar-all-shims findbugs javadoc cobertura checkstyle \
+    -Divy.home=$IVY_HOME -Dhadoop.dist=${COMPILE_HADOOP_DIST} \
+    -Dcobertura.home=${COBERTURA_HOME} -Dcobertura.format=xml \
+    -Dfindbugs.home=${FINDBUGS_HOME} \
+    -Dtest.junit.output.format=xml
+
+if [ "$?" != "0" ]; then
+  echo "Error during compilation phase. Aborting!"
+  exit 1
+fi
+
+# Run second cobertura step on thirdparty tests.
+${ANT} cobertura \
+    -Divy.home=$IVY_HOME -Dtest.junit.output.format=xml \
+    -Dhadoop.dist=${COMPILE_HADOOP_DIST} \
+    -Dcobertura.home=${COBERTURA_HOME} -Dcobertura.format=xml \
+    -Dsqoop.thirdparty.lib.dir=${THIRDPARTY_LIBS} \
+    -Dtestcase=ThirdPartyTests
+
+if [ "$?" != "0" ]; then
+  echo "Unit tests failed!"
+  exit 1
+fi
+

src/scripts/hudson/run-tests.sh

@@ -0,0 +1,65 @@
+#!/bin/bash -x
+#
+# Licensed to Cloudera, Inc. under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Compiles the world and runs available unit tests.
+# This script is intended for execution by users who want to thoroughly
+# execute all tests, or automated testing agents such as Hudson.
+
+# Environment:
+# See test-config.sh
+
+bin=`readlink -f $0`
+bin=`dirname ${bin}`
+bin=`cd ${bin} && pwd`
+source ${bin}/test-config.sh
+
+# Run compilation step.
+
+${ANT} clean jar -Divy.home=$IVY_HOME -Dhadoop.dist=${COMPILE_HADOOP_DIST}
+if [ "$?" != "0" ]; then
+  echo "Error during compilation phase. Aborting!"
+  exit 1
+fi
+
+testfailed=0
+
+# Run basic unit tests.
+
+${ANT} clean-cache test -Divy.home=$IVY_HOME -Dtest.junit.output.format=xml \
+    -Dhadoop.dist=${TEST_HADOOP_DIST}
+if [ "$?" != "0" ]; then
+  testfailed=1
+fi
+
+# Run thirdparty integration unit tests.
+
+if [ "${THIRDPARTY_LIBS}" == "" ]; then
+  echo "Warning: $$THIRDPARTY_LIBS not set."
+fi
+
+${ANT} test -Dthirdparty=true -Dsqoop.thirdparty.lib.dir=${THIRDPARTY_LIBS} \
+    -Dtest.junit.output.format=xml -Divy.home=$IVY_HOME \
+    -Dhadoop.dist=${TEST_HADOOP_DIST}
+if [ "$?" != "0" ]; then
+  testfailed=1
+fi
+
+if [ "$testfailed" != "0" ]; then
+  echo "Unit tests failed!"
+  exit 1
+fi
+

src/scripts/hudson/test-config.sh

@@ -0,0 +1,47 @@
+#!/bin/bash -x
+#
+# Licensed to Cloudera, Inc. under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configures environment variables for run-tests.sh and run-code-quality.sh.
+
+# Environment:
+#
+# ANT - path to 'ant' executable
+# COBERTURA_HOME - The path to Cobertura
+# COMPILE_HADOOP_DIST - Hadoop distribution to use for compilation phases
+#    Values are:
+#      "apache" for Apache trunk
+#      "apache21" for Apache branch 0.21
+#      "cloudera" for CDH 3 beta 2
+# FINDBUGS_HOME - The path to findbugs
+# IVY_HOME - path where ivy should cache objects.
+# TEST_HADOOP_DIST - Hadoop distribution to use for testing phases
+# THIRDPARTY_LIBS - Path to the thirdparty library directory.
+# WORKSPACE - The workspace for running the build. Defaults to the project
+#     root directory.
+
+bin=`readlink -f $0`
+bin=`dirname ${bin}`
+bin=`cd ${bin} && pwd`
+export projroot="${bin}/../../../"
+
+export ANT=${ANT:-/bin/ant}
+export COMPILE_HADOOP_DIST=${COMPILE_HADOOP_DIST:-apache}
+export TEST_HADOOP_DIST=${TEST_HADOOP_DIST:-apache}
+
+export WORKSPACE=${WORKSPACE:-$projroot}
+export IVY_HOME=${IVY_HOME:-$WORKSPACE/.ivy2}
+