github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-02 20:21:30 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-47. NPE if --append is specified with HBase import target.

Browse Source 
AppendUtils checks for missing append source and exits gracefully.

From: Aaron Kimball <aaron@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149964 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Bayer 2011-07-22 20:04:13 +00:00
parent 66c753c2e7
commit beb0b2e1c2
2 changed files with 39 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/java/com/cloudera/sqoop/util/AppendUtils.java
View File

@ -77,7 +77,16 @@ public void append() throws IOException {
int nextPartition = 0;
// Create directory in case
if (!fs.exists(tempDir)) {
// This occurs if there was no source (tmp) dir. This might happen
// if the import was an HBase-target import, but the user specified
// --append anyway. This is a warning, not an error.
LOG.warn("Cannot append files to target dir; no such directory: "
+ tempDir);
return;
}
// Create target directory.
if (!fs.exists(userDestDir)) {
LOG.info("Creating missing output directory - " + userDestDir.getName());
fs.mkdirs(userDestDir);
@ -92,9 +101,8 @@ public void append() throws IOException {
moveFiles(fs, tempDir, userDestDir, nextPartition);
// delete temporary path
LOG.debug("Deleting temporary folder "
+ context.getDestination().getName());
fs.delete(context.getDestination(), true);
LOG.debug("Deleting temporary folder " + tempDir.getName());
fs.delete(tempDir, true);
}
/**
@ -141,6 +149,13 @@ private void moveFiles(FileSystem fs, Path sourceDir, Path targetDir,
numpart.setGroupingUsed(false);
Pattern patt = Pattern.compile("part.*-([0-9][0-9][0-9][0-9][0-9]).*");
FileStatus[] tempFiles = fs.listStatus(sourceDir);
if (null == tempFiles) {
// If we've already checked that the dir exists, and now it can't be
// listed, this is a genuine error (permissions, fs integrity, or other).
throw new IOException("Could not list files from " + sourceDir);
}
// Move and rename files & directories from temporary to target-dir thus
// appending file's next partition
for (FileStatus fileStat : tempFiles) {

20
src/test/com/cloudera/sqoop/TestAppendUtils.java
View File

@ -34,10 +34,13 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.cloudera.sqoop.manager.ImportJobContext;
import com.cloudera.sqoop.testutil.CommonArgs;
import com.cloudera.sqoop.testutil.HsqldbTestServer;
import com.cloudera.sqoop.testutil.ImportJobTestCase;
import com.cloudera.sqoop.tool.ImportTool;
import com.cloudera.sqoop.util.AppendUtils;
/**
* Test that --append works.
@ -261,5 +264,22 @@ public void testAppendToTargetDir() throws IOException {
runAppendTest(args, output);
}
/**
* If the append source does not exist, don't crash.
*/
public void testAppendSrcDoesNotExist() throws IOException {
Configuration conf = new Configuration();
conf.set("fs.default.name", "file:///");
SqoopOptions options = new SqoopOptions(conf);
options.setTableName("meep");
Path missingPath = new Path("doesNotExistForAnyReason");
FileSystem local = FileSystem.getLocal(conf);
assertFalse(local.exists(missingPath));
ImportJobContext importContext = new ImportJobContext("meep", null,
options, missingPath);
AppendUtils utils = new AppendUtils(importContext);
utils.append();
}
}