From ae66d60c5686b3042b5e56a67bac15c7a6c183b4 Mon Sep 17 00:00:00 2001 From: Andrew Bayer Date: Fri, 22 Jul 2011 20:04:23 +0000 Subject: [PATCH] SQOOP-101. Sqoop build to use IVY for dependencies. This change modifies Sqoop build to use IVY for retrieving HBase and Zookeeper dependencies. Along with this update, the version number for HBase and Hadoop have been incremented to match the CDH3 Beta 3 versions. Due to this, a couple of tests had to be modified in order to accommodate the changed behavior of the Hadoop classes. From: Arvind Prabhakar git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149997 13f79535-47bb-0310-9956-ffa450edef68 --- build.xml | 45 +------------------ ivy.xml | 22 +++++---- ivy/ivysettings.xml | 8 +++- ivy/libraries.properties | 14 +----- .../cloudera/sqoop/TestIncrementalImport.java | 38 ++++++++-------- .../com/cloudera/sqoop/TestMultiMaps.java | 3 -- 6 files changed, 42 insertions(+), 88 deletions(-) diff --git a/build.xml b/build.xml index 95735002..ba26eaab 100644 --- a/build.xml +++ b/build.xml @@ -180,47 +180,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -239,7 +198,6 @@ - @@ -247,7 +205,6 @@ - @@ -289,7 +246,7 @@ diff --git a/ivy.xml b/ivy.xml index 9c0c52c6..b16d8835 100644 --- a/ivy.xml +++ b/ivy.xml @@ -15,7 +15,9 @@ See the License for the specific language governing permissions and limitations under the License. --> - + @@ -118,10 +120,6 @@ conf="common->master" /> - - - + + + + + + + + + + diff --git a/ivy/ivysettings.xml b/ivy/ivysettings.xml index c3120327..ce4c5227 100644 --- a/ivy/ivysettings.xml +++ b/ivy/ivysettings.xml @@ -37,8 +37,11 @@ + + value="[organisation]/[module]/[revision]/[artifact]-[revision](-[classifier])"/> @@ -47,6 +50,8 @@ + + diff --git a/ivy/libraries.properties b/ivy/libraries.properties index 34e38e24..94ccd370 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -24,20 +24,10 @@ commons-io.version=1.4 commons-lang.version=2.4 commons-logging.version=1.0.4 -# Apache Hadoop dependency version: use trunk. -hadoop-common.apache.version=0.22.0-SNAPSHOT -hadoop-hdfs.apache.version=0.22.0-SNAPSHOT -hadoop-mapred.apache.version=0.22.0-SNAPSHOT - -# Apache 0.21 release Hadoop dependency version: use 0.21. -hadoop-common.apache21.version=0.21.0-SNAPSHOT -hadoop-hdfs.apache21.version=0.21.0-SNAPSHOT -hadoop-mapred.apache21.version=0.21.0-SNAPSHOT - # Cloudera Distribution dependency version -hadoop-core.cloudera.version=0.20.2-314 +hadoop-core.cloudera.version=0.20.2-737 -hbase.version=0.89.20100621+17 +hbase.version=0.89.20100924-28 zookeeper.version=3.3.1+7 hsqldb.version=1.8.0.10 diff --git a/src/test/com/cloudera/sqoop/TestIncrementalImport.java b/src/test/com/cloudera/sqoop/TestIncrementalImport.java index b01f6f89..96d43097 100644 --- a/src/test/com/cloudera/sqoop/TestIncrementalImport.java +++ b/src/test/com/cloudera/sqoop/TestIncrementalImport.java @@ -19,20 +19,21 @@ package com.cloudera.sqoop; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStreamReader; - +import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import junit.framework.TestCase; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -49,10 +50,6 @@ import com.cloudera.sqoop.tool.ImportTool; import com.cloudera.sqoop.tool.JobTool; -import junit.framework.TestCase; - -import java.sql.Connection; - /** * Test the incremental import functionality. * @@ -110,7 +107,7 @@ private void assertRowCount(String table, int numRows) throws SQLException { s.close(); } catch (SQLException sqlE) { LOG.warn("exception: " + sqlE); - } + } } if (null != rs) { @@ -118,7 +115,7 @@ private void assertRowCount(String table, int numRows) throws SQLException { rs.close(); } catch (SQLException sqlE) { LOG.warn("exception: " + sqlE); - } + } } } } @@ -193,7 +190,7 @@ private void createIdTable(String tableName, int insertRows) } /** - * Create a table with an 'id' column full of integers and a + * Create a table with an 'id' column full of integers and a * last_modified column with timestamps. */ private void createTimestampTable(String tableName, int insertRows, @@ -290,14 +287,15 @@ public void assertSpecificNumber(String tableName, int val) { Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); Path tableDir = new Path(warehouse, tableName); FileStatus [] stats = fs.listStatus(tableDir); - String [] fileNames = new String[stats.length]; + String [] filePaths = new String[stats.length]; for (int i = 0; i < stats.length; i++) { - fileNames[i] = stats[i].getPath().toString(); + filePaths[i] = stats[i].getPath().toString(); } // Read the first file that is not a hidden file. boolean foundVal = false; - for (String fileName : fileNames) { + for (String filePath : filePaths) { + String fileName = new Path(filePath).getName(); if (fileName.startsWith("_") || fileName.startsWith(".")) { continue; } @@ -308,11 +306,11 @@ public void assertSpecificNumber(String tableName, int val) { } BufferedReader r = new BufferedReader( - new InputStreamReader(fs.open(new Path(fileName)))); + new InputStreamReader(fs.open(new Path(filePath)))); try { String s = r.readLine(); if (null == s) { - fail("Unexpected empty file " + fileName + "."); + fail("Unexpected empty file " + filePath + "."); } assertEquals(val, (int) Integer.valueOf(s.trim())); @@ -327,7 +325,7 @@ public void assertSpecificNumber(String tableName, int val) { r.close(); } } - } catch (Exception e) { + } catch (IOException e) { fail("Got unexpected exception: " + StringUtils.stringifyException(e)); } } @@ -374,7 +372,7 @@ private List getArgListForTable(String tableName, boolean commonArgs, args.add("id"); args.add("-m"); args.add("1"); - + return args; } @@ -397,7 +395,7 @@ private void createJob(String jobName, List jobArgs, SqoopOptions options = new SqoopOptions(); options.setConf(conf); Sqoop makeJob = new Sqoop(new JobTool(), conf, options); - + List args = new ArrayList(); args.add("--create"); args.add(jobName); @@ -429,7 +427,7 @@ private void runJob(String jobName, Configuration conf) { SqoopOptions options = new SqoopOptions(); options.setConf(conf); Sqoop runJob = new Sqoop(new JobTool(), conf, options); - + List args = new ArrayList(); args.add("--exec"); args.add(jobName); @@ -553,7 +551,7 @@ public void testFullLastModifiedImport() throws Exception { // Given a table of rows imported in the past, // see that they are imported. final String TABLE_NAME = "fullLastModified"; - Timestamp thePast = new Timestamp(System.currentTimeMillis() - 100); + Timestamp thePast = new Timestamp(System.currentTimeMillis() - 100); createTimestampTable(TABLE_NAME, 10, thePast); List args = getArgListForTable(TABLE_NAME, true, false); diff --git a/src/test/com/cloudera/sqoop/TestMultiMaps.java b/src/test/com/cloudera/sqoop/TestMultiMaps.java index 93a3faa8..3c2da4bc 100644 --- a/src/test/com/cloudera/sqoop/TestMultiMaps.java +++ b/src/test/com/cloudera/sqoop/TestMultiMaps.java @@ -137,9 +137,6 @@ public void runMultiMapTest(String splitByCol, int expectedSum) Configuration conf = new Configuration(); int curSum = 0; - assertTrue("Found only " + paths.size() + " path(s); expected > 1.", - paths.size() > 1); - // We expect multiple files. We need to open all the files and sum up the // first column across all of them. for (Path p : paths) {