github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 07:42:00 +08:00
Code Issues Projects Releases Wiki Activity

MAPREDUCE-816. Rename "local" mysql import to "direct" in Sqoop. (Aaron Kimball via matei).

Browse Source 
From: Matei Alexandru Zaharia <matei@apache.org>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149819 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Bayer 2011-07-22 20:03:22 +00:00
parent 3d39962dfa
commit ecbb765019
7 changed files with 26 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/java/org/apache/hadoop/sqoop/ConnFactory.java
View File

@ -71,7 +71,7 @@ public static ConnManager getManager(ImportOptions opts) throws IOException {
}
if (scheme.equals("jdbc:mysql:")) {
if (opts.isLocal()) {
if (opts.isDirect()) {
return new LocalMySQLManager(opts);
} else {
return new MySQLManager(opts);

22
src/java/org/apache/hadoop/sqoop/ImportOptions.java
View File

@ -93,7 +93,7 @@ public enum FileLayout {
private String driverClassName;
private String warehouseDir;
private FileLayout layout;
private boolean local; // if true and conn is mysql, use mysqldump.
private boolean direct; // if true and conn is mysql, use mysqldump.
private String tmpDir; // where temp data goes; usually /tmp
private String hiveHome;
private boolean hiveImport;
@ -161,10 +161,10 @@ private void loadFromProperties() {
this.className = props.getProperty("java.classname", this.className);
this.packageName = props.getProperty("java.packagename", this.packageName);
String localImport = props.getProperty("local.import",
Boolean.toString(this.local)).toLowerCase();
this.local = "true".equals(localImport) || "yes".equals(localImport)
|| "1".equals(localImport);
String directImport = props.getProperty("direct.import",
Boolean.toString(this.direct)).toLowerCase();
this.direct = "true".equals(directImport) || "yes".equals(directImport)
|| "1".equals(directImport);
String hiveImportStr = props.getProperty("hive.import",
Boolean.toString(this.hiveImport)).toLowerCase();
@ -250,7 +250,7 @@ public static void printUsage() {
System.out.println("--username (username) Set authentication username");
System.out.println("--password (password) Set authentication password");
System.out.println("-P Read password from console");
System.out.println("--local Use local import fast path (mysql only)");
System.out.println("--direct Use direct import fast path (mysql only)");
System.out.println("");
System.out.println("Import control options:");
System.out.println("--table (tablename) Table to read");
@ -418,7 +418,11 @@ public void parse(String [] args) throws InvalidOptionsException {
} else if (args[i].equals("--all-tables")) {
this.allTables = true;
} else if (args[i].equals("--local")) {
this.local = true;
// TODO(aaron): Remove this after suitable deprecation time period.
LOG.warn("--local is deprecated; use --direct instead.");
this.direct = true;
} else if (args[i].equals("--direct")) {
this.direct = true;
} else if (args[i].equals("--username")) {
this.username = args[++i];
if (null == this.password) {
@ -614,8 +618,8 @@ public String getPassword() {
return password;
}
public boolean isLocal() {
return local;
public boolean isDirect() {
return direct;
}
/**

2
src/java/org/apache/hadoop/sqoop/manager/ConnManager.java
View File

@ -65,7 +65,7 @@ public interface ConnManager {
/**
* Execute a SQL statement to read the named set of columns from a table.
* If columns is null, all columns from the table are read. This is a local
* If columns is null, all columns from the table are read. This is a direct
* (non-parallelized) read of the table back to the current client.
* The client is responsible for calling ResultSet.close() when done with the
* returned ResultSet object.

8
src/java/org/apache/hadoop/sqoop/manager/LocalMySQLManager.java
View File

@ -49,9 +49,8 @@
import org.apache.hadoop.util.Shell;
/**
* Manages local connections to MySQL databases
* that are local to this machine -- so we can use mysqldump to get
* really fast dumps.
* Manages direct connections to MySQL databases
* so we can use mysqldump to get really fast dumps.
*/
public class LocalMySQLManager extends MySQLManager {
@ -391,7 +390,7 @@ public void importTable(String tableName, String jarFile, Configuration conf)
// TODO(aaron): Support SequenceFile-based load-in
LOG.warn("File import layout " + options.getFileLayout()
+ " is not supported by");
LOG.warn("MySQL local import; import will proceed as text files.");
LOG.warn("MySQL direct import; import will proceed as text files.");
}
ArrayList<String> args = new ArrayList<String>();
@ -461,6 +460,7 @@ public void importTable(String tableName, String jarFile, Configuration conf)
args.add("--quick"); // no buffering
// TODO(aaron): Add a flag to allow --lock-tables instead for MyISAM data
args.add("--single-transaction");
// TODO(aaron): Add --host and --port arguments to support remote direct connects.
String username = options.getUsername();
if (null != username) {

8
src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java
View File

@ -41,7 +41,7 @@ public class MySQLManager extends GenericJdbcManager {
// driver class to ensure is loaded when making db connection.
private static final String DRIVER_CLASS = "com.mysql.jdbc.Driver";
// set to true after we warn the user that we can use local fastpath.
// set to true after we warn the user that we can use direct fastpath.
private static boolean warningPrinted = false;
public MySQLManager(final ImportOptions opts) {
@ -49,7 +49,7 @@ public MySQLManager(final ImportOptions opts) {
}
protected MySQLManager(final ImportOptions opts, boolean ignored) {
// constructor used by subclasses to avoid the --local warning.
// constructor used by subclasses to avoid the --direct warning.
super(DRIVER_CLASS, opts);
}
@ -102,9 +102,11 @@ public void importTable(String tableName, String jarFile, Configuration conf)
if (null != connectString && connectString.indexOf("//localhost") != -1) {
// if we're not doing a remote connection, they should have a LocalMySQLManager.
// TODO(aaron): After LocalMySQLManager supports --host/--port, this should
// always be issued.
LOG.warn("It looks like you are importing from mysql on");
LOG.warn("localhost. This transfer can be faster! Use the");
LOG.warn("--local option to exercise a MySQL-specific fast");
LOG.warn("--direct option to exercise a MySQL-specific fast");
LOG.warn("path.");
MySQLManager.warningPrinted = true; // don't display this twice.

2
src/test/org/apache/hadoop/sqoop/manager/LocalMySQLTest.java
View File

@ -193,7 +193,7 @@ private String getCurrentUser() {
args.add(getWarehouseDir());
args.add("--connect");
args.add(CONNECT_STRING);
args.add("--local");
args.add("--direct");
args.add("--username");
args.add(getCurrentUser());
args.add("--where");

2
src/test/org/apache/hadoop/sqoop/manager/MySQLAuthTest.java
View File

@ -144,7 +144,7 @@ public void tearDown() {
args.add(getWarehouseDir());
args.add("--connect");
args.add(AUTH_CONNECT_STRING);
args.add("--local");
args.add("--direct");
args.add("--username");
args.add(AUTH_TEST_USER);
args.add("--password");