github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-04 08:19:57 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-332. Cannot use --as-avrodatafile with --query.

Browse Source 
(Joseph Boyd via Arvind Prabhakar)


git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1170977 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arvind Prabhakar 2011-09-15 06:55:33 +00:00
parent e1e6e5c009
commit 4992ce7515
2 changed files with 85 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/java/com/cloudera/sqoop/orm/AvroSchemaGenerator.java
View File

@ -63,10 +63,13 @@ public Schema generate() throws IOException {
field.addProp("sqlType", Integer.toString(sqlType)); field.addProp("sqlType", Integer.toString(sqlType));
fields.add(field); fields.add(field);
} }
String doc = "Sqoop import of " + tableName;
Schema schema = Schema.createRecord(tableName, doc, null, false); String avroTableName = (tableName == null ? "QueryResult" : tableName);
String doc = "Sqoop import of " + avroTableName;
Schema schema = Schema.createRecord(avroTableName, doc, null, false);
schema.setFields(fields); schema.setFields(fields);
schema.addProp("tableName", tableName); schema.addProp("tableName", avroTableName);
return schema; return schema;
} }

97
src/test/com/cloudera/sqoop/TestAvroImportExportRoundtrip.java
View File

@ -44,6 +44,16 @@ public class TestAvroImportExportRoundtrip extends ImportJobTestCase {
public static final Log LOG = LogFactory public static final Log LOG = LogFactory
.getLog(TestAvroImportExportRoundtrip.class.getName()); .getLog(TestAvroImportExportRoundtrip.class.getName());
public void testRoundtripQuery() throws IOException, SQLException {
String[] argv = {};
runImport(getOutputArgvForQuery(true));
deleteTableData();
runExport(getExportArgvForQuery(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
checkFirstColumnSum();
}
public void testRoundtrip() throws IOException, SQLException { public void testRoundtrip() throws IOException, SQLException {
String[] argv = {}; String[] argv = {};
@ -79,6 +89,73 @@ protected String[] getOutputArgv(boolean includeHadoopFlags) {
return args.toArray(new String[0]); return args.toArray(new String[0]);
} }
/**
* Create the argv to pass to Sqoop.
*
* @return the argv as an array of strings.
*/
protected String[] getOutputArgvForQuery(boolean includeHadoopFlags) {
ArrayList<String> args = new ArrayList<String>();
if (includeHadoopFlags) {
CommonArgs.addHadoopFlags(args);
}
args.add("--query");
args.add("select * from " + HsqldbTestServer.getTableName() + " where $CONDITIONS");
args.add("--connect");
args.add(HsqldbTestServer.getUrl());
args.add("--target-dir");
args.add(getWarehouseDir() + "/query_result");
args.add("--split-by");
args.add("INTFIELD1");
args.add("--as-avrodatafile");
return args.toArray(new String[0]);
}
protected String [] getExportArgv(boolean includeHadoopFlags,
int rowsPerStmt, int statementsPerTx, String... additionalArgv) {
ArrayList<String> args = formatAdditionalArgs(additionalArgv);
args.add("--table");
args.add(getTableName());
args.add("--export-dir");
args.add(getTablePath().toString());
args.add("--connect");
args.add(getConnectString());
args.add("-m");
args.add("1");
LOG.debug("args:");
for (String a : args) {
LOG.debug(" " + a);
}
return args.toArray(new String[0]);
}
protected String [] getExportArgvForQuery(boolean includeHadoopFlags,
int rowsPerStmt, int statementsPerTx, String... additionalArgv) {
ArrayList<String> args = formatAdditionalArgs(additionalArgv);
args.add("--table");
args.add(getTableName());
args.add("--export-dir");
args.add(getWarehouseDir() + "/query_result");
args.add("--connect");
args.add(getConnectString());
args.add("-m");
args.add("1");
LOG.debug("args:");
for (String a : args) {
LOG.debug(" " + a);
}
return args.toArray(new String[0]);
}
/** /**
* Create the argv to pass to Sqoop. * Create the argv to pass to Sqoop.
* @param includeHadoopFlags if true, then include -D various.settings=values * @param includeHadoopFlags if true, then include -D various.settings=values
@ -86,8 +163,7 @@ protected String[] getOutputArgv(boolean includeHadoopFlags) {
* @param statementsPerTx ## of statements to use in a transaction. * @param statementsPerTx ## of statements to use in a transaction.
* @return the argv as an array of strings. * @return the argv as an array of strings.
*/ */
protected String [] getExportArgv(boolean includeHadoopFlags, protected ArrayList<String> formatAdditionalArgs(String... additionalArgv) {
int rowsPerStmt, int statementsPerTx, String... additionalArgv) {
ArrayList<String> args = new ArrayList<String>(); ArrayList<String> args = new ArrayList<String>();
// Any additional Hadoop flags (-D foo=bar) are prepended. // Any additional Hadoop flags (-D foo=bar) are prepended.
@ -120,22 +196,7 @@ protected String[] getOutputArgv(boolean includeHadoopFlags) {
} }
} }
} }
return args;
args.add("--table");
args.add(getTableName());
args.add("--export-dir");
args.add(getTablePath().toString());
args.add("--connect");
args.add(getConnectString());
args.add("-m");
args.add("1");
LOG.debug("args:");
for (String a : args) {
LOG.debug(" " + a);
}
return args.toArray(new String[0]);
} }
// this test just uses the two int table. // this test just uses the two int table.