github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 21:31:27 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1392: Create the temporary directory inside task working dir rather then in tmp

Browse Source 
(Richard via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2014-09-01 10:52:15 +02:00
parent e0fc46e949
commit 4ee8b6843a
9 changed files with 12 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/java/com/cloudera/sqoop/lib/LargeObjectLoader.java
View File

@ -47,10 +47,9 @@ public class LargeObjectLoader extends org.apache.sqoop.lib.LargeObjectLoader {
/**
* Create a new LargeObjectLoader.
* @param conf the Configuration to use
* @param workPath the HDFS working directory for this task.
*/
public LargeObjectLoader(Configuration conf, Path workPath)
public LargeObjectLoader(Configuration conf)
throws IOException {
super(conf, workPath);
super(conf);
}
}

5
src/java/org/apache/sqoop/lib/LargeObjectLoader.java
View File

@ -69,12 +69,11 @@ public class LargeObjectLoader implements Closeable {
/**
* Create a new LargeObjectLoader.
* @param conf the Configuration to use
* @param workPath the HDFS working directory for this task.
*/
public LargeObjectLoader(Configuration conf, Path workPath)
public LargeObjectLoader(Configuration conf)
throws IOException {
this.conf = conf;
this.workPath = workPath;
this.workPath = new Path(System.getProperty("java.io.tmpdir"), "SQOOP");
this.fs = FileSystem.get(conf);
this.curBlobWriter = null;
this.curClobWriter = null;

3
src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java
View File

@ -52,8 +52,7 @@ protected void setup(Context context)
throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
schema = AvroJob.getMapOutputSchema(conf);
lobLoader = new LargeObjectLoader(conf,
FileOutputFormat.getWorkOutputPath(context));
lobLoader = new LargeObjectLoader(conf);
bigDecimalFormatString = conf.getBoolean(
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);

5
src/java/org/apache/sqoop/mapreduce/HBaseBulkImportMapper.java
View File

@ -54,10 +54,7 @@ public class HBaseBulkImportMapper
protected void setup(Context context)
throws IOException, InterruptedException {
this.conf = context.getConfiguration();
Path largeFilePath = new Path(this.conf.get("sqoop.hbase.lob.extern.dir",
"/tmp/sqoop-hbase-" + context.getTaskAttemptID()));
this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
largeFilePath);
this.lobLoader = new LargeObjectLoader(this.conf);
// Get the implementation of PutTransformer to use.
// By default, we call toString() on every non-null field.

4
src/java/org/apache/sqoop/mapreduce/ParquetImportMapper.java
View File

@ -42,9 +42,7 @@ public class ParquetImportMapper
@Override
protected void setup(Context context)
throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
Path workPath = new Path("/tmp/sqoop-parquet-" + context.getTaskAttemptID());
lobLoader = new LargeObjectLoader(conf, workPath);
lobLoader = new LargeObjectLoader(context.getConfiguration());
}
@Override

3
src/java/org/apache/sqoop/mapreduce/SequenceFileImportMapper.java
View File

@ -38,8 +38,7 @@ public class SequenceFileImportMapper
@Override
protected void setup(Context context)
throws IOException, InterruptedException {
this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
FileOutputFormat.getWorkOutputPath(context));
this.lobLoader = new LargeObjectLoader(context.getConfiguration());
}
@Override

3
src/java/org/apache/sqoop/mapreduce/TextImportMapper.java
View File

@ -45,8 +45,7 @@ public TextImportMapper() {
@Override
protected void setup(Context context)
throws IOException, InterruptedException {
this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
FileOutputFormat.getWorkOutputPath(context));
this.lobLoader = new LargeObjectLoader(context.getConfiguration());
}
@Override

3
src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java
View File

@ -112,8 +112,7 @@ public SqoopHCatImportHelper(Configuration conf) throws IOException,
hCatFullTableSchema.append(hfs);
}
fieldCount = hCatFullTableSchema.size();
lobLoader = new LargeObjectLoader(conf, new Path(jobInfo.getTableInfo()
.getTableLocation()));
lobLoader = new LargeObjectLoader(conf);
bigDecimalFormatString = conf.getBoolean(
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);

4
src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java
View File

@ -49,14 +49,14 @@ public void setUp() throws IOException, InterruptedException {
conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
}
String tmpDir = System.getProperty("test.build.data", "/tmp/");
this.outDir = new Path(new Path(tmpDir), "testLobLoader");
this.outDir = new Path(System.getProperty("java.io.tmpdir"));
FileSystem fs = FileSystem.get(conf);
if (fs.exists(outDir)) {
fs.delete(outDir, true);
}
fs.mkdirs(outDir);
loader = new LargeObjectLoader(conf, outDir);
loader = new LargeObjectLoader(conf);
}
public void testReadClobRef()