SQOOP-101. Sqoop build to use IVY for dependencies.

This change modifies Sqoop build to use IVY for retrieving HBase and
Zookeeper dependencies. Along with this update, the version number
for HBase and Hadoop have been incremented to match the CDH3 Beta 3
versions. Due to this, a couple of tests had to be modified in order
to accommodate the changed behavior of the Hadoop classes.

From: Arvind Prabhakar <arvind@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149997 13f79535-47bb-0310-9956-ffa450edef68

build.xml

@@ -180,47 +180,6 @@
     </classpath>
   </taskdef>
 
-  <!-- Download/untar HBase and ZooKeeper -->
-  <property name="cdh.tar.dir" value="http://archive.cloudera.com/cdh/3"/>
-  <property name="hbase.tar" value="hbase-${hbase.version}.tar.gz"/>
-  <property name="hbase.tar.url" value="${cdh.tar.dir}/${hbase.tar}"/>
-  <property name="hbase.dir" location="${build.dir}/hbase-${hbase.version}" />
-  <property name="zk.tar" value="zookeeper-${zookeeper.version}.tar.gz"/>
-  <property name="zk.tar.url" value="${cdh.tar.dir}/${zk.tar}"/>
-  <property name="zk.dir" location="${build.dir}/zookeeper-${zookeeper.version}" />
-
- <target name="download-hbase-zk" unless="offline">
-   <get dest="${build.dir}/${hbase.tar}" src="${hbase.tar.url}"
-        usetimestamp="true" verbose="true"/>
-   <get dest="${build.dir}/${zk.tar}" src="${zk.tar.url}" usetimestamp="true"
-        verbose="true"/>
-   <condition property="hbase-zk-already-untared">
-      <and>
-         <available file="${hbase.dir}" type="dir"/>
-         <available file="${zk.dir}" type="dir"/>
-      </and>
-   </condition>
- </target>
-
- <target name="untar-hbase-zk" unless="hbase-zk-already-untared"
-   depends="download-hbase-zk">
-   <untar src="${build.dir}/${hbase.tar}" dest="${build.dir}"
-    compression="gzip"/>
-   <untar src="${build.dir}/${zk.tar}" dest="${build.dir}" compression="gzip"/>
- </target>
-
-  <!-- manually-specified HBase classpath til it works with Ivy. -->
-  <path id="hbase.classpath">
-    <fileset dir="${hbase.dir}">
-      <include name="*.jar" />
-      <include name="lib/*.jar" />
-    </fileset>
-    <fileset dir="${zk.dir}">
-      <include name="*.jar" />
-      <include name="lib/*.jar" />
-    </fileset>
-  </path>
-
   <!-- The classpath for compiling and running Sqoop -->
   <if>
     <isset property="hadoop.home" />
@@ -239,7 +198,6 @@
           <include name="*.jar" />
         </fileset>
         <path refid="${name}.hadoop.classpath"/>
-        <path refid="hbase.classpath"/>
       </path>
     </then>
     <else>
@@ -247,7 +205,6 @@
         <pathelement location="${build.classes}"/>
         <path refid="lib.path"/>
         <path refid="${name}.hadoop.classpath"/>
-        <path refid="hbase.classpath"/>
       </path>
     </else>
   </if>
@@ -289,7 +246,7 @@
 
   <!-- Compile core classes for the project -->
   <target name="compile"
-      depends="init, gen-version, ivy-retrieve-hadoop, untar-hbase-zk"
+      depends="init, gen-version, ivy-retrieve-hadoop"
       description="Compile core classes for the project">
     <!-- don't use an out-of-date instrumented build. -->
     <delete dir="${cobertura.class.dir}" />

ivy.xml

@@ -15,7 +15,9 @@
    See the License for the specific language governing permissions and
    limitations under the License.
 -->
-<ivy-module version="1.0">
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven"
+            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xsi:noNamespaceSchemaLocation="http://ant.apache.org/ivy/schemas/ivy.xsd">
   <info organisation="com.cloudera.sqoop" module="${name}">
     <license name="Apache 2.0"/>
     <ivyauthor name="Sqoop Team" url="http://github.com/cloudera/sqoop" />
@@ -118,10 +120,6 @@
       conf="common->master" />
     <dependency org="junit" name="junit" rev="${junit.version}"
       conf="test->default"/>
-    <!--
-    <dependency org="org.apache.hbase" name="hbase" rev="${hbase.version}"
-      conf="common->default" />
-      -->
     <dependency org="hsqldb" name="hsqldb" rev="${hsqldb.version}"
       conf="common->default"/>
     <dependency org="commons-io" name="commons-io" rev="${commons-io.version}"
@@ -138,8 +136,16 @@
     <dependency org="commons-collections" name="commons-collections"
       rev="${commons-collections.version}" conf="releaseaudit->default"/>
 
-    <!-- dependencies on the shim jars we generate, for release poms -->
+    <dependency org="org.apache.hbase" name="hbase"
-    <dependency org="com.cloudera.sqoop" name="sqoop-shim-cloudera"
+      rev="${hbase.version}" conf="cloudera->default">
-      rev="${version}" conf="mvndeploy->default" />
+      <artifact name="hbase" type="jar"/>
+      <artifact name="hbase" type="test-jar" ext="jar" m:classifier="tests"/>
+      <exclude org="com.sun.jersey" module="jersey-core"/>
+      <exclude org="com.sun.jersey" module="jersey-json"/>
+      <exclude org="com.sun.jersey" module="jersey-server"/>
+      <exclude org="org.apache.thrift" module="thrift"/>
+      <exclude org="log4j" module="log4j"/>
+    </dependency>
+
   </dependencies>
 </ivy-module>

ivy/ivysettings.xml

@@ -37,8 +37,11 @@
   <property name="staging.cloudera.com"
       value="https://repository.cloudera.com/content/repositories/staging/"
       override="false"/>
+  <property name="releases.cloudera.com"
+      value="https://repository.cloudera.com/content/repositories/releases/"
+      override="false"/>
   <property name="maven2.pattern"
-      value="[organisation]/[module]/[revision]/[module]-[revision]"/>
+      value="[organisation]/[module]/[revision]/[artifact]-[revision](-[classifier])"/>
   <property name="repo.dir" value="${user.home}/.m2/repository"/>
   <property name="maven2.pattern.ext"  value="${maven2.pattern}.[ext]"/>
   <property name="resolvers" value="default" override="false"/>
@@ -47,6 +50,8 @@
   <resolvers>
     <ibiblio name="maven2" root="${repo.maven.org}"
         pattern="${maven2.pattern.ext}" m2compatible="true"/>
+    <ibiblio name="cloudera-releases" root="${releases.cloudera.com}"
+        pattern="${maven2.pattern.ext}" m2compatible="true"/>
     <ibiblio name="apache-snapshot" root="${snapshot.apache.org}"
         m2compatible="true" checkmodified="true" changingPattern=".*SNAPSHOT"/>
     <ibiblio name="cloudera-staging" root="${staging.cloudera.com}"
@@ -60,6 +65,7 @@
     <chain name="default" dual="true" checkmodified="true"
         changingPattern=".*SNAPSHOT">
       <resolver ref="apache-snapshot"/> 
+      <resolver ref="cloudera-releases"/>
       <resolver ref="cloudera-staging"/> 
       <resolver ref="maven2"/>
     </chain>

ivy/libraries.properties

@@ -24,20 +24,10 @@ commons-io.version=1.4
 commons-lang.version=2.4
 commons-logging.version=1.0.4
 
-# Apache Hadoop dependency version: use trunk.
-hadoop-common.apache.version=0.22.0-SNAPSHOT
-hadoop-hdfs.apache.version=0.22.0-SNAPSHOT
-hadoop-mapred.apache.version=0.22.0-SNAPSHOT
-
-# Apache 0.21 release Hadoop dependency version: use 0.21.
-hadoop-common.apache21.version=0.21.0-SNAPSHOT
-hadoop-hdfs.apache21.version=0.21.0-SNAPSHOT
-hadoop-mapred.apache21.version=0.21.0-SNAPSHOT
-
 # Cloudera Distribution dependency version
-hadoop-core.cloudera.version=0.20.2-314
+hadoop-core.cloudera.version=0.20.2-737
 
-hbase.version=0.89.20100621+17
+hbase.version=0.89.20100924-28
 zookeeper.version=3.3.1+7
 
 hsqldb.version=1.8.0.10

src/test/com/cloudera/sqoop/TestIncrementalImport.java

@@ -19,20 +19,21 @@
 package com.cloudera.sqoop;
 
 import java.io.BufferedReader;
+import java.io.IOException;
 import java.io.InputStreamReader;
-
+import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Timestamp;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import junit.framework.TestCase;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -49,10 +50,6 @@
 import com.cloudera.sqoop.tool.ImportTool;
 import com.cloudera.sqoop.tool.JobTool;
 
-import junit.framework.TestCase;
-
-import java.sql.Connection;
-
 /**
  * Test the incremental import functionality.
  *
@@ -110,7 +107,7 @@ private void assertRowCount(String table, int numRows) throws SQLException {
           s.close();
         } catch (SQLException sqlE) {
           LOG.warn("exception: " + sqlE);
-        } 
+        }
       }
 
       if (null != rs) {
@@ -118,7 +115,7 @@ private void assertRowCount(String table, int numRows) throws SQLException {
           rs.close();
         } catch (SQLException sqlE) {
           LOG.warn("exception: " + sqlE);
-        } 
+        }
       }
     }
   }
@@ -193,7 +190,7 @@ private void createIdTable(String tableName, int insertRows)
   }
 
   /**
-   * Create a table with an 'id' column full of integers and a 
+   * Create a table with an 'id' column full of integers and a
    * last_modified column with timestamps.
    */
   private void createTimestampTable(String tableName, int insertRows,
@@ -290,14 +287,15 @@ public void assertSpecificNumber(String tableName, int val) {
       Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
       Path tableDir = new Path(warehouse, tableName);
       FileStatus [] stats = fs.listStatus(tableDir);
-      String [] fileNames = new String[stats.length];
+      String [] filePaths = new String[stats.length];
       for (int i = 0; i < stats.length; i++) {
-        fileNames[i] = stats[i].getPath().toString();
+        filePaths[i] = stats[i].getPath().toString();
       }
 
       // Read the first file that is not a hidden file.
       boolean foundVal = false;
-      for (String fileName : fileNames) {
+      for (String filePath : filePaths) {
+        String fileName = new Path(filePath).getName();
         if (fileName.startsWith("_") || fileName.startsWith(".")) {
           continue;
         }
@@ -308,11 +306,11 @@ public void assertSpecificNumber(String tableName, int val) {
         }
 
         BufferedReader r = new BufferedReader(
-            new InputStreamReader(fs.open(new Path(fileName))));
+            new InputStreamReader(fs.open(new Path(filePath))));
         try {
           String s = r.readLine();
           if (null == s) {
-            fail("Unexpected empty file " + fileName + ".");
+            fail("Unexpected empty file " + filePath + ".");
           }
           assertEquals(val, (int) Integer.valueOf(s.trim()));
 
@@ -327,7 +325,7 @@ public void assertSpecificNumber(String tableName, int val) {
           r.close();
         }
       }
-    } catch (Exception e) {
+    } catch (IOException e) {
       fail("Got unexpected exception: " + StringUtils.stringifyException(e));
     }
   }
@@ -374,7 +372,7 @@ private List<String> getArgListForTable(String tableName, boolean commonArgs,
     args.add("id");
     args.add("-m");
     args.add("1");
-    
+
     return args;
   }
 
@@ -397,7 +395,7 @@ private void createJob(String jobName, List<String> jobArgs,
       SqoopOptions options = new SqoopOptions();
       options.setConf(conf);
       Sqoop makeJob = new Sqoop(new JobTool(), conf, options);
-      
+
       List<String> args = new ArrayList<String>();
       args.add("--create");
       args.add(jobName);
@@ -429,7 +427,7 @@ private void runJob(String jobName, Configuration conf) {
       SqoopOptions options = new SqoopOptions();
       options.setConf(conf);
       Sqoop runJob = new Sqoop(new JobTool(), conf, options);
-      
+
       List<String> args = new ArrayList<String>();
       args.add("--exec");
       args.add(jobName);
@@ -553,7 +551,7 @@ public void testFullLastModifiedImport() throws Exception {
     // Given a table of rows imported in the past,
     // see that they are imported.
     final String TABLE_NAME = "fullLastModified";
-    Timestamp thePast = new Timestamp(System.currentTimeMillis() - 100); 
+    Timestamp thePast = new Timestamp(System.currentTimeMillis() - 100);
     createTimestampTable(TABLE_NAME, 10, thePast);
 
     List<String> args = getArgListForTable(TABLE_NAME, true, false);

src/test/com/cloudera/sqoop/TestMultiMaps.java

@@ -137,9 +137,6 @@ public void runMultiMapTest(String splitByCol, int expectedSum)
       Configuration conf = new Configuration();
       int curSum = 0;
 
-      assertTrue("Found only " + paths.size() + " path(s); expected > 1.",
-          paths.size() > 1);
-
       // We expect multiple files. We need to open all the files and sum up the
       // first column across all of them.
       for (Path p : paths) {