mirror of
https://github.com/apache/sqoop.git
synced 2025-05-04 14:50:00 +08:00
SQOOP-1392: Create the temporary directory inside task working dir rather then in tmp
(Richard via Jarek Jarcec Cecho)
This commit is contained in:
parent
e0fc46e949
commit
4ee8b6843a
@ -47,10 +47,9 @@ public class LargeObjectLoader extends org.apache.sqoop.lib.LargeObjectLoader {
|
|||||||
/**
|
/**
|
||||||
* Create a new LargeObjectLoader.
|
* Create a new LargeObjectLoader.
|
||||||
* @param conf the Configuration to use
|
* @param conf the Configuration to use
|
||||||
* @param workPath the HDFS working directory for this task.
|
|
||||||
*/
|
*/
|
||||||
public LargeObjectLoader(Configuration conf, Path workPath)
|
public LargeObjectLoader(Configuration conf)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
super(conf, workPath);
|
super(conf);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -69,12 +69,11 @@ public class LargeObjectLoader implements Closeable {
|
|||||||
/**
|
/**
|
||||||
* Create a new LargeObjectLoader.
|
* Create a new LargeObjectLoader.
|
||||||
* @param conf the Configuration to use
|
* @param conf the Configuration to use
|
||||||
* @param workPath the HDFS working directory for this task.
|
|
||||||
*/
|
*/
|
||||||
public LargeObjectLoader(Configuration conf, Path workPath)
|
public LargeObjectLoader(Configuration conf)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
this.workPath = workPath;
|
this.workPath = new Path(System.getProperty("java.io.tmpdir"), "SQOOP");
|
||||||
this.fs = FileSystem.get(conf);
|
this.fs = FileSystem.get(conf);
|
||||||
this.curBlobWriter = null;
|
this.curBlobWriter = null;
|
||||||
this.curClobWriter = null;
|
this.curClobWriter = null;
|
||||||
|
@ -52,8 +52,7 @@ protected void setup(Context context)
|
|||||||
throws IOException, InterruptedException {
|
throws IOException, InterruptedException {
|
||||||
Configuration conf = context.getConfiguration();
|
Configuration conf = context.getConfiguration();
|
||||||
schema = AvroJob.getMapOutputSchema(conf);
|
schema = AvroJob.getMapOutputSchema(conf);
|
||||||
lobLoader = new LargeObjectLoader(conf,
|
lobLoader = new LargeObjectLoader(conf);
|
||||||
FileOutputFormat.getWorkOutputPath(context));
|
|
||||||
bigDecimalFormatString = conf.getBoolean(
|
bigDecimalFormatString = conf.getBoolean(
|
||||||
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
|
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
|
||||||
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);
|
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);
|
||||||
|
@ -54,10 +54,7 @@ public class HBaseBulkImportMapper
|
|||||||
protected void setup(Context context)
|
protected void setup(Context context)
|
||||||
throws IOException, InterruptedException {
|
throws IOException, InterruptedException {
|
||||||
this.conf = context.getConfiguration();
|
this.conf = context.getConfiguration();
|
||||||
Path largeFilePath = new Path(this.conf.get("sqoop.hbase.lob.extern.dir",
|
this.lobLoader = new LargeObjectLoader(this.conf);
|
||||||
"/tmp/sqoop-hbase-" + context.getTaskAttemptID()));
|
|
||||||
this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
|
|
||||||
largeFilePath);
|
|
||||||
|
|
||||||
// Get the implementation of PutTransformer to use.
|
// Get the implementation of PutTransformer to use.
|
||||||
// By default, we call toString() on every non-null field.
|
// By default, we call toString() on every non-null field.
|
||||||
|
@ -42,9 +42,7 @@ public class ParquetImportMapper
|
|||||||
@Override
|
@Override
|
||||||
protected void setup(Context context)
|
protected void setup(Context context)
|
||||||
throws IOException, InterruptedException {
|
throws IOException, InterruptedException {
|
||||||
Configuration conf = context.getConfiguration();
|
lobLoader = new LargeObjectLoader(context.getConfiguration());
|
||||||
Path workPath = new Path("/tmp/sqoop-parquet-" + context.getTaskAttemptID());
|
|
||||||
lobLoader = new LargeObjectLoader(conf, workPath);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -38,8 +38,7 @@ public class SequenceFileImportMapper
|
|||||||
@Override
|
@Override
|
||||||
protected void setup(Context context)
|
protected void setup(Context context)
|
||||||
throws IOException, InterruptedException {
|
throws IOException, InterruptedException {
|
||||||
this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
|
this.lobLoader = new LargeObjectLoader(context.getConfiguration());
|
||||||
FileOutputFormat.getWorkOutputPath(context));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -45,8 +45,7 @@ public TextImportMapper() {
|
|||||||
@Override
|
@Override
|
||||||
protected void setup(Context context)
|
protected void setup(Context context)
|
||||||
throws IOException, InterruptedException {
|
throws IOException, InterruptedException {
|
||||||
this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
|
this.lobLoader = new LargeObjectLoader(context.getConfiguration());
|
||||||
FileOutputFormat.getWorkOutputPath(context));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -112,8 +112,7 @@ public SqoopHCatImportHelper(Configuration conf) throws IOException,
|
|||||||
hCatFullTableSchema.append(hfs);
|
hCatFullTableSchema.append(hfs);
|
||||||
}
|
}
|
||||||
fieldCount = hCatFullTableSchema.size();
|
fieldCount = hCatFullTableSchema.size();
|
||||||
lobLoader = new LargeObjectLoader(conf, new Path(jobInfo.getTableInfo()
|
lobLoader = new LargeObjectLoader(conf);
|
||||||
.getTableLocation()));
|
|
||||||
bigDecimalFormatString = conf.getBoolean(
|
bigDecimalFormatString = conf.getBoolean(
|
||||||
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
|
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
|
||||||
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);
|
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);
|
||||||
|
@ -49,14 +49,14 @@ public void setUp() throws IOException, InterruptedException {
|
|||||||
conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
|
conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
|
||||||
}
|
}
|
||||||
String tmpDir = System.getProperty("test.build.data", "/tmp/");
|
String tmpDir = System.getProperty("test.build.data", "/tmp/");
|
||||||
this.outDir = new Path(new Path(tmpDir), "testLobLoader");
|
this.outDir = new Path(System.getProperty("java.io.tmpdir"));
|
||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
if (fs.exists(outDir)) {
|
if (fs.exists(outDir)) {
|
||||||
fs.delete(outDir, true);
|
fs.delete(outDir, true);
|
||||||
}
|
}
|
||||||
fs.mkdirs(outDir);
|
fs.mkdirs(outDir);
|
||||||
|
|
||||||
loader = new LargeObjectLoader(conf, outDir);
|
loader = new LargeObjectLoader(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testReadClobRef()
|
public void testReadClobRef()
|
||||||
|
Loading…
Reference in New Issue
Block a user