github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-21 19:31:13 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-89. Support multiple output files in ManagerCompatTestCase.

Browse Source 
From: Aaron Kimball <aaron@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149979 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Bayer 2011-07-22 20:04:17 +00:00
parent c7b1ddb708
commit ea716f5426
1 changed files with 44 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/test/com/cloudera/sqoop/testutil/ImportJobTestCase.java
View File

@ -18,7 +18,7 @@
package com.cloudera.sqoop.testutil;
import java.io.File;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@ -26,6 +26,8 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.Sqoop;
@ -108,7 +110,7 @@ protected List<String> getExtraArgs(Configuration conf) {
protected void verifyImport(String expectedVal, String [] importCols) {
// paths to where our output file will wind up.
Path dataFilePath = getDataFilePath();
Path tableDirPath = getTablePath();
removeTableDir();
@ -144,21 +146,47 @@ protected void verifyImport(String expectedVal, String [] importCols) {
prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
getTableName());
// now actually open the file and check it.
File f = new File(dataFilePath.toString());
assertTrue("Error: " + dataFilePath.toString() + " does not exist",
f.exists());
// Now open and check all part-files in the table path until we find
// a non-empty one that we can verify contains the value.
Object readValue = SeqFileReader.getFirstValue(dataFilePath.toString());
FileSystem fs = FileSystem.getLocal(conf);
FileStatus [] stats = fs.listStatus(tableDirPath);
if (stats == null || stats.length == 0) {
fail("Error: no files in " + tableDirPath);
}
boolean foundRecord = false;
for (FileStatus stat : stats) {
if (!stat.getPath().getName().startsWith("part-")
&& !stat.getPath().getName().startsWith("data-")) {
// This isn't a data file. Ignore it.
continue;
}
try {
Object readValue = SeqFileReader.getFirstValue(
stat.getPath().toString());
LOG.info("Read back from sequencefile: " + readValue);
foundRecord = true;
// Add trailing '\n' to expected value since SqoopRecord.toString()
// encodes the record delim.
if (null == expectedVal) {
assertEquals("Error validating result from SeqFile", "null\n",
readValue.toString());
} else {
assertEquals("Error validating result from SeqFile", expectedVal + "\n",
readValue.toString());
assertEquals("Error validating result from SeqFile",
expectedVal + "\n", readValue.toString());
}
} catch (EOFException eoe) {
// EOF in a file isn't necessarily a problem. We may have some
// empty sequence files, which will throw this. Just continue
// in the loop.
}
}
if (!foundRecord) {
fail("Couldn't read any records from SequenceFiles");
}
} catch (IOException ioe) {
fail("IOException: " + ioe.toString());