5
0
mirror of https://github.com/apache/sqoop.git synced 2025-05-04 10:30:02 +08:00

SQOOP-1392: Create the temporary directory inside task working dir rather then in tmp

(Richard via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2014-09-01 10:52:15 +02:00
parent e0fc46e949
commit 4ee8b6843a
9 changed files with 12 additions and 23 deletions

View File

@ -47,10 +47,9 @@ public class LargeObjectLoader extends org.apache.sqoop.lib.LargeObjectLoader {
/** /**
* Create a new LargeObjectLoader. * Create a new LargeObjectLoader.
* @param conf the Configuration to use * @param conf the Configuration to use
* @param workPath the HDFS working directory for this task.
*/ */
public LargeObjectLoader(Configuration conf, Path workPath) public LargeObjectLoader(Configuration conf)
throws IOException { throws IOException {
super(conf, workPath); super(conf);
} }
} }

View File

@ -69,12 +69,11 @@ public class LargeObjectLoader implements Closeable {
/** /**
* Create a new LargeObjectLoader. * Create a new LargeObjectLoader.
* @param conf the Configuration to use * @param conf the Configuration to use
* @param workPath the HDFS working directory for this task.
*/ */
public LargeObjectLoader(Configuration conf, Path workPath) public LargeObjectLoader(Configuration conf)
throws IOException { throws IOException {
this.conf = conf; this.conf = conf;
this.workPath = workPath; this.workPath = new Path(System.getProperty("java.io.tmpdir"), "SQOOP");
this.fs = FileSystem.get(conf); this.fs = FileSystem.get(conf);
this.curBlobWriter = null; this.curBlobWriter = null;
this.curClobWriter = null; this.curClobWriter = null;

View File

@ -52,8 +52,7 @@ protected void setup(Context context)
throws IOException, InterruptedException { throws IOException, InterruptedException {
Configuration conf = context.getConfiguration(); Configuration conf = context.getConfiguration();
schema = AvroJob.getMapOutputSchema(conf); schema = AvroJob.getMapOutputSchema(conf);
lobLoader = new LargeObjectLoader(conf, lobLoader = new LargeObjectLoader(conf);
FileOutputFormat.getWorkOutputPath(context));
bigDecimalFormatString = conf.getBoolean( bigDecimalFormatString = conf.getBoolean(
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT, ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT); ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);

View File

@ -54,10 +54,7 @@ public class HBaseBulkImportMapper
protected void setup(Context context) protected void setup(Context context)
throws IOException, InterruptedException { throws IOException, InterruptedException {
this.conf = context.getConfiguration(); this.conf = context.getConfiguration();
Path largeFilePath = new Path(this.conf.get("sqoop.hbase.lob.extern.dir", this.lobLoader = new LargeObjectLoader(this.conf);
"/tmp/sqoop-hbase-" + context.getTaskAttemptID()));
this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
largeFilePath);
// Get the implementation of PutTransformer to use. // Get the implementation of PutTransformer to use.
// By default, we call toString() on every non-null field. // By default, we call toString() on every non-null field.

View File

@ -42,9 +42,7 @@ public class ParquetImportMapper
@Override @Override
protected void setup(Context context) protected void setup(Context context)
throws IOException, InterruptedException { throws IOException, InterruptedException {
Configuration conf = context.getConfiguration(); lobLoader = new LargeObjectLoader(context.getConfiguration());
Path workPath = new Path("/tmp/sqoop-parquet-" + context.getTaskAttemptID());
lobLoader = new LargeObjectLoader(conf, workPath);
} }
@Override @Override

View File

@ -38,8 +38,7 @@ public class SequenceFileImportMapper
@Override @Override
protected void setup(Context context) protected void setup(Context context)
throws IOException, InterruptedException { throws IOException, InterruptedException {
this.lobLoader = new LargeObjectLoader(context.getConfiguration(), this.lobLoader = new LargeObjectLoader(context.getConfiguration());
FileOutputFormat.getWorkOutputPath(context));
} }
@Override @Override

View File

@ -45,8 +45,7 @@ public TextImportMapper() {
@Override @Override
protected void setup(Context context) protected void setup(Context context)
throws IOException, InterruptedException { throws IOException, InterruptedException {
this.lobLoader = new LargeObjectLoader(context.getConfiguration(), this.lobLoader = new LargeObjectLoader(context.getConfiguration());
FileOutputFormat.getWorkOutputPath(context));
} }
@Override @Override

View File

@ -112,8 +112,7 @@ public SqoopHCatImportHelper(Configuration conf) throws IOException,
hCatFullTableSchema.append(hfs); hCatFullTableSchema.append(hfs);
} }
fieldCount = hCatFullTableSchema.size(); fieldCount = hCatFullTableSchema.size();
lobLoader = new LargeObjectLoader(conf, new Path(jobInfo.getTableInfo() lobLoader = new LargeObjectLoader(conf);
.getTableLocation()));
bigDecimalFormatString = conf.getBoolean( bigDecimalFormatString = conf.getBoolean(
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT, ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT); ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);

View File

@ -49,14 +49,14 @@ public void setUp() throws IOException, InterruptedException {
conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
} }
String tmpDir = System.getProperty("test.build.data", "/tmp/"); String tmpDir = System.getProperty("test.build.data", "/tmp/");
this.outDir = new Path(new Path(tmpDir), "testLobLoader"); this.outDir = new Path(System.getProperty("java.io.tmpdir"));
FileSystem fs = FileSystem.get(conf); FileSystem fs = FileSystem.get(conf);
if (fs.exists(outDir)) { if (fs.exists(outDir)) {
fs.delete(outDir, true); fs.delete(outDir, true);
} }
fs.mkdirs(outDir); fs.mkdirs(outDir);
loader = new LargeObjectLoader(conf, outDir); loader = new LargeObjectLoader(conf);
} }
public void testReadClobRef() public void testReadClobRef()