github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-20 19:00:48 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-901: Allow user to override hardcoded boolean strings in direct PostgreSQL connector

Browse Source 
(Jarcec Cecho via Cheolsoo Park)
This commit is contained in:
Cheolsoo Park 2013-02-23 19:31:59 -08:00
parent 0affc4ba9f
commit 8c3c78b11c
4 changed files with 125 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/docs/user/connectors.txt
View File

@ -98,6 +98,22 @@ Argument Description
Default is "public".
---------------------------------------------------------------------------------
The direct connector (used when specified +\--direct+ parameter), offers also
additional extra arguments:
.Additional supported PostgreSQL extra arguments in direct mode:
[grid="all"]
`----------------------------------------`---------------------------------------
Argument Description
---------------------------------------------------------------------------------
+\--boolean-true-string <str>+ String that will be used to encode \
+true+ value of +boolean+ columns.
Default is "TRUE".
+\--boolean-false-string <str>+ String that will be used to encode \
+false+ value of +boolean+ columns.
Default is "FALSE".
---------------------------------------------------------------------------------
Schema support
^^^^^^^^^^^^^^

69
src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
View File

@ -30,9 +30,12 @@
import java.util.List;
import java.util.Map;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sqoop.cli.RelatedOptions;
import org.apache.sqoop.util.PostgreSQLUtils;
import com.cloudera.sqoop.SqoopOptions;
@ -54,15 +57,33 @@
*/
public class DirectPostgresqlManager
extends com.cloudera.sqoop.manager.PostgresqlManager {
public static final Log LOG = LogFactory.getLog(
DirectPostgresqlManager.class.getName());
public static final String BOOLEAN_TRUE_STRING = "boolean-true-string";
public static final String DEFAULT_BOOLEAN_TRUE_STRING = "TRUE";
public static final String BOOLEAN_FALSE_STRING = "boolean-false-string";
public static final String DEFAULT_BOOLEAN_FALSE_STRING = "FALSE";
public DirectPostgresqlManager(final SqoopOptions opts) {
super(opts);
if (this.booleanFalseString == null) {
this.booleanFalseString = DEFAULT_BOOLEAN_FALSE_STRING;
}
if (booleanTrueString == null) {
this.booleanTrueString = DEFAULT_BOOLEAN_TRUE_STRING;
}
}
private static final String PSQL_CMD = "psql";
private String booleanTrueString;
private String booleanFalseString;
/** Copies data directly into HDFS, adding the user's chosen line terminator
char to each record.
*/
@ -196,12 +217,18 @@ private String getSelectListColumnsStr(String [] cols, String tableName) {
sb.append(col);
} else {
if ("bool".equalsIgnoreCase(columnTypes.get(col))) {
sb.append(String.format("case when %s=true then 'TRUE' "
+ "when %s=false then 'FALSE' end as %s",
sb.append(String.format("case when %s=true then "
+ "'" + booleanTrueString + "' "
+ "when %s=false then "
+ "'" + booleanFalseString + "'"
+ " end as %s",
colEscaped, colEscaped, colEscaped));
} else if ("bit".equalsIgnoreCase(columnTypes.get(col))) {
sb.append(String.format("case when %s=B'1' then 'TRUE' "
+ "when %s=B'0' then 'FALSE' end as %s",
sb.append(String.format("case when %s=B'1' then "
+ "'" + booleanTrueString + "' "
+ "when %s=B'0' then "
+ "'" + booleanFalseString + "'"
+ " end as %s",
colEscaped, colEscaped, colEscaped));
} else {
sb.append(colEscaped);
@ -508,4 +535,38 @@ public boolean supportsStagingForExport() {
return false;
}
// CHECKSTYLE:ON
/** {@inheritDoc}. */
@Override
protected void applyExtraArguments(CommandLine cmdLine) {
super.applyExtraArguments(cmdLine);
if (cmdLine.hasOption(BOOLEAN_TRUE_STRING)) {
String arg = cmdLine.getOptionValue(BOOLEAN_TRUE_STRING);
LOG.info("Loaded TRUE encoding string " + arg);
this.booleanTrueString = arg;
}
if (cmdLine.hasOption(BOOLEAN_FALSE_STRING)) {
String arg = cmdLine.getOptionValue(BOOLEAN_FALSE_STRING);
LOG.info("Loaded FALSE encoding string " + arg);
this.booleanFalseString = arg;
}
}
/** {@inheritDoc}. */
@Override
@SuppressWarnings("static-access")
protected RelatedOptions getExtraOptions() {
RelatedOptions extraOptions = super.getExtraOptions();
extraOptions.addOption(OptionBuilder.withArgName("string").hasArg()
.withDescription("String to encode TRUE value")
.withLongOpt(BOOLEAN_TRUE_STRING).create());
extraOptions.addOption(OptionBuilder.withArgName("string").hasArg()
.withDescription("String to encode FALSE value")
.withLongOpt(BOOLEAN_FALSE_STRING).create());
return extraOptions;
}
}

9
src/java/org/apache/sqoop/manager/PostgresqlManager.java
View File

@ -196,7 +196,7 @@ protected String getCurTimestampQuery() {
* @param args Extra arguments array
* @throws ParseException
*/
void parseExtraArgs(String[] args) throws ParseException {
private void parseExtraArgs(String[] args) throws ParseException {
// No-op when no extra arguments are present
if (args == null || args.length == 0) {
return;
@ -207,6 +207,11 @@ void parseExtraArgs(String[] args) throws ParseException {
CommandLineParser parser = new GnuParser();
CommandLine cmdLine = parser.parse(getExtraOptions(), args, true);
// Apply parsed arguments
applyExtraArguments(cmdLine);
}
protected void applyExtraArguments(CommandLine cmdLine) {
// Apply extra options
if (cmdLine.hasOption(SCHEMA)) {
String schemaName = cmdLine.getOptionValue(SCHEMA);
@ -222,7 +227,7 @@ void parseExtraArgs(String[] args) throws ParseException {
* @return
*/
@SuppressWarnings("static-access")
private RelatedOptions getExtraOptions() {
protected RelatedOptions getExtraOptions() {
// Connection args (common)
RelatedOptions extraOptions =
new RelatedOptions("PostgreSQL extra options:");

57
src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
View File

@ -139,12 +139,13 @@ public void setUpData(String tableName, String schema, boolean nullEntry) {
connection.commit();
} catch (SQLException e) {
LOG.info("Couldn't create schema " + schema + " (is o.k. as long as"
+ "the schema already exists.", e);
+ "the schema already exists.");
connection.rollback();
}
String fullTableName = manager.escapeTableName(schema)
+ "." + manager.escapeTableName(tableName);
LOG.info("Creating table: " + fullTableName);
try {
// Try to remove the table first. DROP TABLE IF EXISTS didn't
@ -152,8 +153,7 @@ public void setUpData(String tableName, String schema, boolean nullEntry) {
// any exception here if one occurs.
st.executeUpdate("DROP TABLE " + fullTableName);
} catch (SQLException e) {
LOG.info("Couldn't drop table " + schema + "." + tableName + " (ok)",
e);
LOG.info("Couldn't drop table " + schema + "." + tableName + " (ok)");
// Now we need to reset the transaction.
connection.rollback();
}
@ -163,17 +163,18 @@ public void setUpData(String tableName, String schema, boolean nullEntry) {
+ manager.escapeColName("name") + " VARCHAR(24) NOT NULL, "
+ manager.escapeColName("start_date") + " DATE, "
+ manager.escapeColName("Salary") + " FLOAT, "
+ manager.escapeColName("Fired") + " BOOL, "
+ manager.escapeColName("dept") + " VARCHAR(32))");
st.executeUpdate("INSERT INTO " + fullTableName
+ " VALUES(1,'Aaron','2009-05-14',1000000.00,'engineering')");
+ " VALUES(1,'Aaron','2009-05-14',1000000.00,TRUE,'engineering')");
st.executeUpdate("INSERT INTO " + fullTableName
+ " VALUES(2,'Bob','2009-04-20',400.00,'sales')");
+ " VALUES(2,'Bob','2009-04-20',400.00,TRUE,'sales')");
st.executeUpdate("INSERT INTO " + fullTableName
+ " VALUES(3,'Fred','2009-01-23',15.00,'marketing')");
+ " VALUES(3,'Fred','2009-01-23',15.00,FALSE,'marketing')");
if (nullEntry) {
st.executeUpdate("INSERT INTO " + fullTableName
+ " VALUES(4,'Mike',NULL,NULL,NULL)");
+ " VALUES(4,'Mike',NULL,NULL,NULL,NULL)");
}
connection.commit();
@ -276,8 +277,8 @@ private void doImportAndVerify(boolean isDirect, String [] expectedResults,
@Test
public void testJdbcBasedImport() throws IOException {
String [] expectedResults = {
"2,Bob,2009-04-20,400.0,sales",
"3,Fred,2009-01-23,15.0,marketing",
"2,Bob,2009-04-20,400.0,true,sales",
"3,Fred,2009-01-23,15.0,false,marketing",
};
doImportAndVerify(false, expectedResults, TABLE_NAME);
@ -286,8 +287,8 @@ public void testJdbcBasedImport() throws IOException {
@Test
public void testDirectImport() throws IOException {
String [] expectedResults = {
"2,Bob,2009-04-20,400,sales",
"3,Fred,2009-01-23,15,marketing",
"2,Bob,2009-04-20,400,TRUE,sales",
"3,Fred,2009-01-23,15,FALSE,marketing",
};
doImportAndVerify(true, expectedResults, TABLE_NAME);
@ -309,8 +310,8 @@ public void testListTables() throws IOException {
@Test
public void testTableNameWithSpecialCharacter() throws IOException {
String [] expectedResults = {
"2,Bob,2009-04-20,400.0,sales",
"3,Fred,2009-01-23,15.0,marketing",
"2,Bob,2009-04-20,400.0,true,sales",
"3,Fred,2009-01-23,15.0,false,marketing",
};
doImportAndVerify(false, expectedResults, SPECIAL_TABLE_NAME);
@ -330,8 +331,8 @@ public void testIncrementalImport() throws IOException {
@Test
public void testDifferentSchemaImport() throws IOException {
String [] expectedResults = {
"2,Bob,2009-04-20,400.0,sales",
"3,Fred,2009-01-23,15.0,marketing",
"2,Bob,2009-04-20,400.0,true,sales",
"3,Fred,2009-01-23,15.0,false,marketing",
};
String [] extraArgs = { "--",
@ -344,8 +345,8 @@ public void testDifferentSchemaImport() throws IOException {
@Test
public void testDifferentSchemaImportDirect() throws IOException {
String [] expectedResults = {
"2,Bob,2009-04-20,400,sales",
"3,Fred,2009-01-23,15,marketing",
"2,Bob,2009-04-20,400,TRUE,sales",
"3,Fred,2009-01-23,15,FALSE,marketing",
};
String [] extraArgs = { "--",
@ -358,9 +359,9 @@ public void testDifferentSchemaImportDirect() throws IOException {
@Test
public void testNullEscapeCharacters() throws Exception {
String [] expectedResults = {
"2,Bob,2009-04-20,400,sales",
"3,Fred,2009-01-23,15,marketing",
"4,Mike,\\N,\\N,\\N",
"2,Bob,2009-04-20,400,TRUE,sales",
"3,Fred,2009-01-23,15,FALSE,marketing",
"4,Mike,\\N,\\N,\\N,\\N",
};
String [] extraArgs = {
@ -370,4 +371,20 @@ public void testNullEscapeCharacters() throws Exception {
doImportAndVerify(true, expectedResults, NULL_TABLE_NAME, extraArgs);
}
@Test
public void testDifferentBooleanValues() throws Exception {
String [] expectedResults = {
"2,Bob,2009-04-20,400,REAL_TRUE,sales",
"3,Fred,2009-01-23,15,REAL_FALSE,marketing",
};
String [] extraArgs = {
"--",
"--boolean-true-string", "REAL_TRUE",
"--boolean-false-string", "REAL_FALSE",
};
doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs);
}
}