github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 05:31:14 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-386. Namespace migration cleanup.

Browse Source 
(Bilung Lee via Arvind Prabhakar)


git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1196486 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arvind Prabhakar 2011-11-02 07:48:56 +00:00
parent fd844db769
commit 1598c5a481
181 changed files with 2580 additions and 864 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
NOTICE.txt
View File

@ -1,3 +1,10 @@
This product includes software developed by Cloudera, Inc.
(http://www.cloudera.com/).
Apache SQOOP
Copyright 2011 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
Portions of this software were developed at
Cloudera, Inc. (http://www.cloudera.com/).

20
src/docs/dev/SqoopDevGuide.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@ -24,6 +22,24 @@ Sqoop Developer's Guide v{revnumber}
----
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
----
include::intro.txt[]
include::preface.txt[]

2
src/docs/dev/api-reference.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/dev/compiling.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/dev/intro.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/dev/preface.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/man/codegen-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/man/common-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/man/hbase-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/man/hive-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/man/import-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/man/input-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/man/output-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/sip/INDEX.txt
View File

@ -1,6 +1,4 @@
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/sip/README.txt
View File

@ -1,6 +1,4 @@
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/sip/sip-1.txt
View File

@ -1,6 +1,4 @@
== <!--
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/sip/sip-2.txt
View File

@ -1,6 +1,4 @@
== <!--
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/sip/sip-3.txt
View File

@ -1,6 +1,4 @@
== <!--
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/sip/sip-4.txt
View File

@ -1,6 +1,4 @@
== <!--
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/sip/sip-6.txt
View File

@ -1,6 +1,4 @@
== <!--
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

20
src/docs/user/SqoopUserGuide.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@ -24,6 +22,24 @@ Sqoop User Guide (v{revnumber})
----
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
----
include::intro.txt[]
include::preface.txt[]

2
src/docs/user/basics.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/codegen-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/codegen-purpose.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/codegen.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/common-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/compatibility.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/connecting.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/controlling-input-format.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/controlling-output-format.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/create-hive-table-purpose.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/create-hive-table.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/eval-purpose.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/eval.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/export-purpose.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/export.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/hbase-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/hbase.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/help.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/hive-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/hive-notes.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/hive.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/import-all-tables-purpose.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/import-all-tables.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/import-purpose.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/import.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/input-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/input-formatting-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/input-formatting.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/intro.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/job-purpose.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/list-databases.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/list-tables.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/merge-purpose.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/metastore-purpose.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/output-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/output-formatting-args.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/output-formatting.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/preface.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/saved-jobs.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/support.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/tools.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/troubleshooting.txt
View File

@ -1,6 +1,4 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/user/version.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/web/breadcrumbs.xsl
View File

@ -4,8 +4,6 @@
exclude-result-prefixes="exsl">
<!--
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/web/docbook.css
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information

2
src/docs/web/footer.xsl
View File

@ -4,8 +4,6 @@
exclude-result-prefixes="exsl">
<!--
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/web/header.xsl
View File

@ -4,8 +4,6 @@
exclude-result-prefixes="exsl">
<!--
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

2
src/docs/web/html.xsl
View File

@ -2,8 +2,6 @@
<!--
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information

20
src/docs/web/index.txt
View File

@ -1,7 +1,5 @@
////
Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@ -24,6 +22,24 @@ Sqoop Documentation (v{revnumber})
----
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
----
The following pages are the documentation for Sqoop v{revnumber}:
- link:SqoopUserGuide.html[Sqoop User Guide]

212
src/java/com/cloudera/sqoop/ConnFactory.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -20,215 +18,23 @@
package com.cloudera.sqoop;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import com.cloudera.sqoop.manager.ConnManager;
import com.cloudera.sqoop.manager.DefaultManagerFactory;
import com.cloudera.sqoop.manager.ManagerFactory;
import com.cloudera.sqoop.metastore.JobData;
import com.cloudera.sqoop.util.ClassLoaderStack;
/**
* Factory class to create the ConnManager type required
* for the current import job.
*
* This class delegates the actual responsibility for instantiating
* ConnManagers to one or more instances of ManagerFactory. ManagerFactories
* are consulted in the order specified in sqoop-site.xml
* (sqoop.connection.factories).
* @deprecated Moving to use org.apache.sqoop namespace.
*/
public class ConnFactory {
public class ConnFactory
extends org.apache.sqoop.ConnFactory {
public static final Log LOG = LogFactory.getLog(ConnFactory.class.getName());
public static final String FACTORY_CLASS_NAMES_KEY =
org.apache.sqoop.ConnFactory.FACTORY_CLASS_NAMES_KEY;
public static final String DEFAULT_FACTORY_CLASS_NAMES =
org.apache.sqoop.ConnFactory.DEFAULT_FACTORY_CLASS_NAMES;
public ConnFactory(Configuration conf) {
factories = new LinkedList<ManagerFactory>();
instantiateFactories(conf);
super(conf);
}
/** The sqoop-site.xml configuration property used to set the list of
* available ManagerFactories.
*/
public static final String FACTORY_CLASS_NAMES_KEY =
"sqoop.connection.factories";
// The default value for sqoop.connection.factories is the
// name of the DefaultManagerFactory.
static final String DEFAULT_FACTORY_CLASS_NAMES =
DefaultManagerFactory.class.getName();
/** The list of ManagerFactory instances consulted by getManager().
*/
private List<ManagerFactory> factories;
/**
* Create the ManagerFactory instances that should populate
* the factories list.
*/
private void instantiateFactories(Configuration conf) {
loadManagersFromConfDir(conf);
String [] classNameArray =
conf.getStrings(FACTORY_CLASS_NAMES_KEY, DEFAULT_FACTORY_CLASS_NAMES);
for (String className : classNameArray) {
try {
className = className.trim(); // Ignore leading/trailing whitespace.
ManagerFactory factory = ReflectionUtils.newInstance(
(Class<? extends ManagerFactory>)
conf.getClassByName(className), conf);
LOG.debug("Loaded manager factory: " + className);
factories.add(factory);
} catch (ClassNotFoundException cnfe) {
LOG.error("Could not load ManagerFactory " + className
+ " (not found)");
}
}
}
/**
* Factory method to get a ConnManager for the given JDBC connect string.
* @param data the connection and other configuration arguments.
* @return a ConnManager instance for the appropriate database.
* @throws IOException if it cannot find a ConnManager for this schema.
*/
public ConnManager getManager(JobData data) throws IOException {
// Try all the available manager factories.
for (ManagerFactory factory : factories) {
LOG.debug("Trying ManagerFactory: " + factory.getClass().getName());
ConnManager mgr = factory.accept(data);
if (null != mgr) {
LOG.debug("Instantiated ConnManager " + mgr.toString());
return mgr;
}
}
throw new IOException("No manager for connect string: "
+ data.getSqoopOptions().getConnectString());
}
/**
* Add a ManagerFactory class to the list that we instantiate.
* @param conf the Configuration to set.
* @param factory the ManagerFactory class name to add.
*/
private void addManager(Configuration conf, String factory) {
String curVal = conf.get(FACTORY_CLASS_NAMES_KEY);
if (null == curVal) {
conf.set(FACTORY_CLASS_NAMES_KEY, factory);
} else {
conf.set(FACTORY_CLASS_NAMES_KEY, curVal + "," + factory);
}
}
/**
* Read the specified file and extract any ManagerFactory implementation
* names from there.
* @param conf the configuration to populate.
* @param f the file containing the configuration data to add.
*/
private void addManagersFromFile(Configuration conf, File f) {
Reader r = null;
try {
// The file format is actually Java properties-file syntax.
r = new InputStreamReader(new FileInputStream(f));
Properties props = new Properties();
props.load(r);
for (Map.Entry<Object, Object> entry : props.entrySet()) {
// Each key is a ManagerFactory class name.
// Each value, if set, is the jar that contains it.
String factory = entry.getKey().toString();
addManager(conf, factory);
String jarName = entry.getValue().toString();
if (jarName.length() > 0) {
ClassLoaderStack.addJarFile(jarName, factory);
LOG.debug("Added factory " + factory + " in jar " + jarName
+ " specified by " + f);
} else if (LOG.isDebugEnabled()) {
LOG.debug("Added factory " + factory + " specified by " + f);
}
}
} catch (IOException ioe) {
LOG.error("Error loading ManagerFactory information from file "
+ f + ": " + StringUtils.stringifyException(ioe));
} finally {
if (null != r) {
try {
r.close();
} catch (IOException ioe) {
LOG.warn("Error closing file " + f + ": " + ioe);
}
}
}
}
/**
* If $SQOOP_CONF_DIR/managers.d/ exists and sqoop.connection.factories is
* not set, then we look through the files in that directory; they should
* contain lines of the form mgr.class.name[=/path/to/containing.jar].
*
* <p>
* Put all mgr.class.names into the Configuration, and load any specified
* jars into the ClassLoader.
* </p>
*
* @param conf the current configuration to populate with class names.
* @return conf again, after possibly populating sqoop.connection.factories.
*/
private Configuration loadManagersFromConfDir(Configuration conf) {
if (conf.get(FACTORY_CLASS_NAMES_KEY) != null) {
LOG.debug(FACTORY_CLASS_NAMES_KEY + " is set; ignoring managers.d");
return conf;
}
String confDirName = System.getenv("SQOOP_CONF_DIR");
if (null == confDirName) {
LOG.warn("$SQOOP_CONF_DIR has not been set in the environment. "
+ "Cannot check for additional configuration.");
return conf;
}
File confDir = new File(confDirName);
File mgrDir = new File(confDir, "managers.d");
if (mgrDir.exists() && mgrDir.isDirectory()) {
// We have a managers.d subdirectory. Get the file list, sort it,
// and process them in order.
String [] fileNames = mgrDir.list();
Arrays.sort(fileNames);
for (String fileName : fileNames) {
File f = new File(mgrDir, fileName);
if (f.isFile()) {
addManagersFromFile(conf, f);
}
}
// Add the default MF.
addManager(conf, DEFAULT_FACTORY_CLASS_NAMES);
}
// Set the classloader in this configuration so that it will use
// the jars we just loaded in.
conf.setClassLoader(Thread.currentThread().getContextClassLoader());
return conf;
}
}

222
src/java/com/cloudera/sqoop/Sqoop.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -20,224 +18,56 @@
package com.cloudera.sqoop;
import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import com.cloudera.sqoop.cli.ToolOptions;
import com.cloudera.sqoop.tool.SqoopTool;
import com.cloudera.sqoop.util.OptionsFileUtil;
/**
* Main entry-point for Sqoop
* Usage: hadoop jar (this_jar_name) com.cloudera.sqoop.Sqoop (options)
* See the SqoopOptions class for options.
* @deprecated Moving to use org.apache.sqoop namespace.
*/
public class Sqoop extends Configured implements Tool {
public class Sqoop
extends org.apache.sqoop.Sqoop {
public static final Log SQOOP_LOG = LogFactory.getLog("com.cloudera.sqoop");
public static final Log LOG = LogFactory.getLog(Sqoop.class.getName());
public static final Log SQOOP_LOG =
org.apache.sqoop.Sqoop.SQOOP_LOG;
/**
* If this System property is set, always throw an exception, do not just
* exit with status 1.
*/
public static final String SQOOP_RETHROW_PROPERTY = "sqoop.throwOnError";
public static final String SQOOP_RETHROW_PROPERTY =
org.apache.sqoop.Sqoop.SQOOP_RETHROW_PROPERTY;
/**
* The option to specify an options file from which other options to the
* tool are read.
*/
public static final String SQOOP_OPTIONS_FILE_SPECIFIER = "--options-file";
public static final String SQOOP_OPTIONS_FILE_SPECIFIER =
org.apache.sqoop.Sqoop.SQOOP_OPTIONS_FILE_SPECIFIER;
static {
Configuration.addDefaultResource("sqoop-site.xml");
}
private SqoopTool tool;
private SqoopOptions options;
private String [] childPrgmArgs;
/**
* Creates a new instance of Sqoop set to run the supplied SqoopTool
* with the default configuration.
* @param tool the SqoopTool to run in the main body of Sqoop.
*/
public Sqoop(SqoopTool tool) {
this(tool, (Configuration) null);
}
/**
* Creates a new instance of Sqoop set to run the supplied SqoopTool
* with the provided configuration.
* @param tool the SqoopTool to run in the main body of Sqoop.
* @param conf the Configuration to use (e.g., from ToolRunner).
*/
public Sqoop(SqoopTool tool, Configuration conf) {
this(tool, conf, new SqoopOptions());
}
/**
* Creates a new instance of Sqoop set to run the supplied SqoopTool
* with the provided configuration and SqoopOptions.
* @param tool the SqoopTool to run in the main body of Sqoop.
* @param conf the Configuration to use (e.g., from ToolRunner).
* @param opts the SqoopOptions which control the tool's parameters.
*/
public Sqoop(SqoopTool tool, Configuration conf, SqoopOptions opts) {
if (null != conf) {
setConf(conf);
}
this.options = opts;
this.options.setConf(getConf());
this.tool = tool;
}
/**
* @return the SqoopOptions used in this Sqoop instance.
*/
public SqoopOptions getOptions() {
return this.options;
}
/**
* @return the SqoopTool used in this Sqoop instance.
*/
public SqoopTool getTool() {
return this.tool;
}
@Override
/**
* Actual main entry-point for the program
*/
public int run(String [] args) {
if (options.getConf() == null) {
// Configuration wasn't initialized until after the ToolRunner
// got us to this point. ToolRunner gave Sqoop itself a Conf
// though.
options.setConf(getConf());
}
try {
options = tool.parseArguments(args, null, options, false);
tool.appendArgs(this.childPrgmArgs);
tool.validateOptions(options);
} catch (Exception e) {
// Couldn't parse arguments.
// Log the stack trace for this exception
LOG.debug(e.getMessage(), e);
// Print exception message.
System.err.println(e.getMessage());
// Print the tool usage message and exit.
ToolOptions toolOpts = new ToolOptions();
tool.configureOptions(toolOpts);
tool.printHelp(toolOpts);
return 1; // Exit on exception here.
}
return tool.run(options);
}
/**
* SqoopTools sometimes pass arguments to a child program (e.g., mysqldump).
* Users can specify additional args to these programs by preceeding the
* additional arguments with a standalone '--'; but
* ToolRunner/GenericOptionsParser will cull out this argument. We remove
* the child-program arguments in advance, and store them to be readded
* later.
* @param argv the argv in to the SqoopTool
* @return the argv with a "--" and any subsequent arguments removed.
*/
private String [] stashChildPrgmArgs(String [] argv) {
for (int i = 0; i < argv.length; i++) {
if ("--".equals(argv[i])) {
this.childPrgmArgs = Arrays.copyOfRange(argv, i, argv.length);
return Arrays.copyOfRange(argv, 0, i);
}
}
// Didn't find child-program arguments.
return argv;
}
/**
* Given a Sqoop object and a set of arguments to deliver to
* its embedded SqoopTool, run the tool, wrapping the call to
* ToolRunner.
* This entry-point is preferred to ToolRunner.run() because
* it has a chance to stash child program arguments before
* GenericOptionsParser would remove them.
*/
public static int runSqoop(Sqoop sqoop, String [] args) {
try {
String [] toolArgs = sqoop.stashChildPrgmArgs(args);
return ToolRunner.run(sqoop.getConf(), sqoop, toolArgs);
} catch (Exception e) {
LOG.error("Got exception running Sqoop: " + e.toString());
e.printStackTrace();
if (System.getProperty(SQOOP_RETHROW_PROPERTY) != null) {
throw new RuntimeException(e);
}
return 1;
}
return org.apache.sqoop.Sqoop.runSqoop(sqoop, args);
}
/**
* Entry-point that parses the correct SqoopTool to use from the args,
* but does not call System.exit() as main() will.
*/
public static int runTool(String [] args, Configuration conf) {
// Expand the options
String[] expandedArgs = null;
try {
expandedArgs = OptionsFileUtil.expandArguments(args);
} catch (Exception ex) {
LOG.error("Error while expanding arguments", ex);
System.err.println(ex.getMessage());
System.err.println("Try 'sqoop help' for usage.");
return 1;
}
String toolName = expandedArgs[0];
Configuration pluginConf = SqoopTool.loadPlugins(conf);
SqoopTool tool = SqoopTool.getTool(toolName);
if (null == tool) {
System.err.println("No such sqoop tool: " + toolName
+ ". See 'sqoop help'.");
return 1;
}
Sqoop sqoop = new Sqoop(tool, pluginConf);
return runSqoop(sqoop,
Arrays.copyOfRange(expandedArgs, 1, expandedArgs.length));
return org.apache.sqoop.Sqoop.runTool(args, conf);
}
/**
* Entry-point that parses the correct SqoopTool to use from the args,
* but does not call System.exit() as main() will.
*/
public static int runTool(String [] args) {
return runTool(args, new Configuration());
return org.apache.sqoop.Sqoop.runTool(args);
}
public static void main(String [] args) {
if (args.length == 0) {
System.err.println("Try 'sqoop help' for usage.");
System.exit(1);
}
int ret = runTool(args);
System.exit(ret);
org.apache.sqoop.Sqoop.main(args);
}
public Sqoop(SqoopTool tool) {
super(tool);
}
public Sqoop(SqoopTool tool, Configuration conf) {
super(tool, conf);
}
public Sqoop(SqoopTool tool, Configuration conf, SqoopOptions opts) {
super(tool, conf, opts);
}
}

2
src/java/com/cloudera/sqoop/SqoopOptions.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information

2
src/java/com/cloudera/sqoop/cli/RelatedOptions.java
View File

@ -20,7 +20,7 @@
/**
* @deprecated Moving to use org.apache.sqoop namespace.
*/
public class RelatedOptions
public class RelatedOptions
extends org.apache.sqoop.cli.RelatedOptions {
public RelatedOptions() {

12
src/java/com/cloudera/sqoop/hbase/ToStringPutTransformer.java
View File

@ -18,18 +18,6 @@
package com.cloudera.sqoop.hbase;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
/**
* @deprecated Moving to use org.apache.sqoop namespace.
*/

2
src/java/com/cloudera/sqoop/io/LobFile.java
View File

@ -44,7 +44,7 @@ private LobFile() {
org.apache.sqoop.io.LobFile.LATEST_LOB_VERSION;
// Must be in sync with org.apache.sqoop.io.LobFile.HEADER_ID_STR
static final char [] HEADER_ID_STR =
static final char [] HEADER_ID_STR =
org.apache.sqoop.io.LobFile.HEADER_ID_STR;
// Value for entryId to write to the beginning of an IndexSegment.

128
src/java/com/cloudera/sqoop/manager/PostgresqlManager.java
View File

@ -18,140 +18,20 @@
package com.cloudera.sqoop.manager;
import java.io.IOException;
import java.sql.SQLException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.util.ImportException;
/**
* Manages connections to Postgresql databases.
*/
public class PostgresqlManager extends CatalogQueryManager {
public static final Log LOG = LogFactory.getLog(
PostgresqlManager.class.getName());
// driver class to ensure is loaded when making db connection.
private static final String DRIVER_CLASS = "org.postgresql.Driver";
// set to true after we warn the user that we can use direct fastpath.
private static boolean warningPrinted = false;
public class PostgresqlManager
extends org.apache.sqoop.manager.PostgresqlManager {
public PostgresqlManager(final SqoopOptions opts) {
super(DRIVER_CLASS, opts);
super(opts);
}
protected PostgresqlManager(final SqoopOptions opts, boolean ignored) {
// constructor used by subclasses to avoid the --direct warning.
super(DRIVER_CLASS, opts);
}
@Override
public String escapeColName(String colName) {
return escapeIdentifier(colName);
}
@Override
public String escapeTableName(String tableName) {
return escapeIdentifier(tableName);
}
protected String escapeIdentifier(String identifier) {
if (identifier == null) {
return null;
}
return "\"" + identifier.replace("\"", "\"\"") + "\"";
}
@Override
public void close() throws SQLException {
if (this.hasOpenConnection()) {
this.getConnection().commit(); // Commit any changes made thus far.
}
super.close();
}
@Override
protected String getColNamesQuery(String tableName) {
// Use LIMIT to return fast
return "SELECT t.* FROM " + escapeTableName(tableName) + " AS t LIMIT 1";
}
@Override
public void importTable(ImportJobContext context)
throws IOException, ImportException {
// The user probably should have requested --direct to invoke pg_dump.
// Display a warning informing them of this fact.
if (!PostgresqlManager.warningPrinted) {
LOG.warn("It looks like you are importing from postgresql.");
LOG.warn("This transfer can be faster! Use the --direct");
LOG.warn("option to exercise a postgresql-specific fast path.");
PostgresqlManager.warningPrinted = true; // don't display this twice.
}
// Then run the normal importTable() method.
super.importTable(context);
}
@Override
public boolean supportsStagingForExport() {
return true;
}
@Override
protected String getListDatabasesQuery() {
return
"SELECT DATNAME FROM PG_CATALOG.PG_DATABASE";
}
@Override
protected String getListTablesQuery() {
return
"SELECT TABLENAME FROM PG_CATALOG.PG_TABLES "
+ "WHERE SCHEMANAME = (SELECT CURRENT_SCHEMA())";
}
@Override
protected String getListColumnsQuery(String tableName) {
return
"SELECT col.ATTNAME FROM PG_CATALOG.PG_NAMESPACE sch,"
+ " PG_CATALOG.PG_CLASS tab, PG_CATALOG.PG_ATTRIBUTE col "
+ "WHERE sch.OID = tab.RELNAMESPACE "
+ " AND tab.OID = col.ATTRELID "
+ " AND sch.NSPNAME = (SELECT CURRENT_SCHEMA()) "
+ " AND tab.RELNAME = '" + escapeLiteral(tableName) + "' "
+ " AND col.ATTNUM >= 1";
}
@Override
protected String getPrimaryKeyQuery(String tableName) {
return
"SELECT col.ATTNAME FROM PG_CATALOG.PG_NAMESPACE sch, "
+ " PG_CATALOG.PG_CLASS tab, PG_CATALOG.PG_ATTRIBUTE col, "
+ " PG_CATALOG.PG_INDEX ind "
+ "WHERE sch.OID = tab.RELNAMESPACE "
+ " AND tab.OID = col.ATTRELID "
+ " AND tab.OID = ind.INDRELID "
+ " AND sch.NSPNAME = (SELECT CURRENT_SCHEMA()) "
+ " AND tab.RELNAME = '" + escapeLiteral(tableName) + "' "
+ " AND col.ATTNUM = ANY(ind.INDKEY) "
+ " AND ind.INDISPRIMARY";
}
private String escapeLiteral(String literal) {
return literal.replace("'", "''");
}
@Override
protected String getCurTimestampQuery() {
return "SELECT CURRENT_TIMESTAMP";
super(opts, ignored);
}
}

2
src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information

2
src/java/com/cloudera/sqoop/mapreduce/DelegatingOutputFormat.java
View File

@ -24,7 +24,7 @@
* @deprecated Moving to use org.apache.sqoop namespace.
*/
public class DelegatingOutputFormat<K extends FieldMappable, V>
extends org.apache.sqoop.mapreduce.DelegatingOutputFormat <K, V> {
extends org.apache.sqoop.mapreduce.DelegatingOutputFormat<K, V> {
public static final String DELEGATE_CLASS_KEY =
org.apache.sqoop.mapreduce.DelegatingOutputFormat.DELEGATE_CLASS_KEY;

2
src/java/com/cloudera/sqoop/mapreduce/ExportBatchOutputFormat.java
View File

@ -25,5 +25,5 @@
*/
public class ExportBatchOutputFormat<K extends SqoopRecord, V>
extends org.apache.sqoop.mapreduce.ExportBatchOutputFormat
<SqoopRecord,V> {
<SqoopRecord, V> {
}

1
src/java/com/cloudera/sqoop/mapreduce/db/DBOutputFormat.java
View File

@ -21,7 +21,6 @@
import java.sql.PreparedStatement;
import java.sql.SQLException;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
/**

2
src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
View File

@ -47,7 +47,7 @@ public class DataDrivenDBInputFormat<T extends DBWritable>
* @deprecated use org.apache.sqoop.mapreduce.db.DataDrivenDBInputFormat.
* DataDrivenDBInputSplit instead.
* @see org.apache.sqoop.mapreduce.db.DataDrivenDBInputFormat.
* DataDrivenDBInputSplit
* DataDrivenDBInputSplit
*/
public static class DataDrivenDBInputSplit extends
org.apache.sqoop.mapreduce.db.DataDrivenDBInputFormat.DataDrivenDBInputSplit {

2
src/java/com/cloudera/sqoop/tool/CodeGenTool.java
View File

@ -20,6 +20,6 @@
/**
* @deprecated Moving to use org.apache.sqoop namespace.
*/
public class CodeGenTool
public class CodeGenTool
extends org.apache.sqoop.tool.CodeGenTool {
}

2
src/java/com/cloudera/sqoop/tool/SqoopTool.java
View File

@ -19,8 +19,6 @@
import java.util.Set;
import org.apache.hadoop.util.StringUtils;
/**
* @deprecated Moving to use org.apache.sqoop namespace.
*/

2
src/java/com/cloudera/sqoop/util/TaskId.java
View File

@ -23,8 +23,6 @@
import org.apache.hadoop.conf.Configuration;
import com.cloudera.sqoop.config.ConfigurationConstants;
/**
* @deprecated Moving to use org.apache.sqoop namespace.
*/

232
src/java/org/apache/sqoop/ConnFactory.java Normal file
View File

@ -0,0 +1,232 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import com.cloudera.sqoop.manager.ConnManager;
import com.cloudera.sqoop.manager.DefaultManagerFactory;
import com.cloudera.sqoop.manager.ManagerFactory;
import com.cloudera.sqoop.metastore.JobData;
import com.cloudera.sqoop.util.ClassLoaderStack;
/**
* Factory class to create the ConnManager type required
* for the current import job.
*
* This class delegates the actual responsibility for instantiating
* ConnManagers to one or more instances of ManagerFactory. ManagerFactories
* are consulted in the order specified in sqoop-site.xml
* (sqoop.connection.factories).
*/
public class ConnFactory {
public static final Log LOG = LogFactory.getLog(ConnFactory.class.getName());
public ConnFactory(Configuration conf) {
factories = new LinkedList<ManagerFactory>();
instantiateFactories(conf);
}
/** The sqoop-site.xml configuration property used to set the list of
* available ManagerFactories.
*/
public static final String FACTORY_CLASS_NAMES_KEY =
"sqoop.connection.factories";
// The default value for sqoop.connection.factories is the
// name of the DefaultManagerFactory.
public static final String DEFAULT_FACTORY_CLASS_NAMES =
DefaultManagerFactory.class.getName();
/** The list of ManagerFactory instances consulted by getManager().
*/
private List<ManagerFactory> factories;
/**
* Create the ManagerFactory instances that should populate
* the factories list.
*/
private void instantiateFactories(Configuration conf) {
loadManagersFromConfDir(conf);
String [] classNameArray =
conf.getStrings(FACTORY_CLASS_NAMES_KEY, DEFAULT_FACTORY_CLASS_NAMES);
for (String className : classNameArray) {
try {
className = className.trim(); // Ignore leading/trailing whitespace.
ManagerFactory factory = ReflectionUtils.newInstance(
(Class<? extends ManagerFactory>)
conf.getClassByName(className), conf);
LOG.debug("Loaded manager factory: " + className);
factories.add(factory);
} catch (ClassNotFoundException cnfe) {
LOG.error("Could not load ManagerFactory " + className
+ " (not found)");
}
}
}
/**
* Factory method to get a ConnManager for the given JDBC connect string.
* @param data the connection and other configuration arguments.
* @return a ConnManager instance for the appropriate database.
* @throws IOException if it cannot find a ConnManager for this schema.
*/
public ConnManager getManager(JobData data) throws IOException {
// Try all the available manager factories.
for (ManagerFactory factory : factories) {
LOG.debug("Trying ManagerFactory: " + factory.getClass().getName());
ConnManager mgr = factory.accept(data);
if (null != mgr) {
LOG.debug("Instantiated ConnManager " + mgr.toString());
return mgr;
}
}
throw new IOException("No manager for connect string: "
+ data.getSqoopOptions().getConnectString());
}
/**
* Add a ManagerFactory class to the list that we instantiate.
* @param conf the Configuration to set.
* @param factory the ManagerFactory class name to add.
*/
private void addManager(Configuration conf, String factory) {
String curVal = conf.get(FACTORY_CLASS_NAMES_KEY);
if (null == curVal) {
conf.set(FACTORY_CLASS_NAMES_KEY, factory);
} else {
conf.set(FACTORY_CLASS_NAMES_KEY, curVal + "," + factory);
}
}
/**
* Read the specified file and extract any ManagerFactory implementation
* names from there.
* @param conf the configuration to populate.
* @param f the file containing the configuration data to add.
*/
private void addManagersFromFile(Configuration conf, File f) {
Reader r = null;
try {
// The file format is actually Java properties-file syntax.
r = new InputStreamReader(new FileInputStream(f));
Properties props = new Properties();
props.load(r);
for (Map.Entry<Object, Object> entry : props.entrySet()) {
// Each key is a ManagerFactory class name.
// Each value, if set, is the jar that contains it.
String factory = entry.getKey().toString();
addManager(conf, factory);
String jarName = entry.getValue().toString();
if (jarName.length() > 0) {
ClassLoaderStack.addJarFile(jarName, factory);
LOG.debug("Added factory " + factory + " in jar " + jarName
+ " specified by " + f);
} else if (LOG.isDebugEnabled()) {
LOG.debug("Added factory " + factory + " specified by " + f);
}
}
} catch (IOException ioe) {
LOG.error("Error loading ManagerFactory information from file "
+ f + ": " + StringUtils.stringifyException(ioe));
} finally {
if (null != r) {
try {
r.close();
} catch (IOException ioe) {
LOG.warn("Error closing file " + f + ": " + ioe);
}
}
}
}
/**
* If $SQOOP_CONF_DIR/managers.d/ exists and sqoop.connection.factories is
* not set, then we look through the files in that directory; they should
* contain lines of the form mgr.class.name[=/path/to/containing.jar].
*
* <p>
* Put all mgr.class.names into the Configuration, and load any specified
* jars into the ClassLoader.
* </p>
*
* @param conf the current configuration to populate with class names.
* @return conf again, after possibly populating sqoop.connection.factories.
*/
private Configuration loadManagersFromConfDir(Configuration conf) {
if (conf.get(FACTORY_CLASS_NAMES_KEY) != null) {
LOG.debug(FACTORY_CLASS_NAMES_KEY + " is set; ignoring managers.d");
return conf;
}
String confDirName = System.getenv("SQOOP_CONF_DIR");
if (null == confDirName) {
LOG.warn("$SQOOP_CONF_DIR has not been set in the environment. "
+ "Cannot check for additional configuration.");
return conf;
}
File confDir = new File(confDirName);
File mgrDir = new File(confDir, "managers.d");
if (mgrDir.exists() && mgrDir.isDirectory()) {
// We have a managers.d subdirectory. Get the file list, sort it,
// and process them in order.
String [] fileNames = mgrDir.list();
Arrays.sort(fileNames);
for (String fileName : fileNames) {
File f = new File(mgrDir, fileName);
if (f.isFile()) {
addManagersFromFile(conf, f);
}
}
// Add the default MF.
addManager(conf, DEFAULT_FACTORY_CLASS_NAMES);
}
// Set the classloader in this configuration so that it will use
// the jars we just loaded in.
conf.setClassLoader(Thread.currentThread().getContextClassLoader());
return conf;
}
}

242
src/java/org/apache/sqoop/Sqoop.java Normal file
View File

@ -0,0 +1,242 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop;
import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.cli.ToolOptions;
import com.cloudera.sqoop.tool.SqoopTool;
import com.cloudera.sqoop.util.OptionsFileUtil;
/**
* Main entry-point for Sqoop
* Usage: hadoop jar (this_jar_name) com.cloudera.sqoop.Sqoop (options)
* See the SqoopOptions class for options.
*/
public class Sqoop extends Configured implements Tool {
public static final Log SQOOP_LOG = LogFactory.getLog("com.cloudera.sqoop");
public static final Log LOG = LogFactory.getLog(Sqoop.class.getName());
/**
* If this System property is set, always throw an exception, do not just
* exit with status 1.
*/
public static final String SQOOP_RETHROW_PROPERTY = "sqoop.throwOnError";
/**
* The option to specify an options file from which other options to the
* tool are read.
*/
public static final String SQOOP_OPTIONS_FILE_SPECIFIER = "--options-file";
static {
Configuration.addDefaultResource("sqoop-site.xml");
}
private SqoopTool tool;
private SqoopOptions options;
private String [] childPrgmArgs;
/**
* Creates a new instance of Sqoop set to run the supplied SqoopTool
* with the default configuration.
* @param tool the SqoopTool to run in the main body of Sqoop.
*/
public Sqoop(SqoopTool tool) {
this(tool, (Configuration) null);
}
/**
* Creates a new instance of Sqoop set to run the supplied SqoopTool
* with the provided configuration.
* @param tool the SqoopTool to run in the main body of Sqoop.
* @param conf the Configuration to use (e.g., from ToolRunner).
*/
public Sqoop(SqoopTool tool, Configuration conf) {
this(tool, conf, new SqoopOptions());
}
/**
* Creates a new instance of Sqoop set to run the supplied SqoopTool
* with the provided configuration and SqoopOptions.
* @param tool the SqoopTool to run in the main body of Sqoop.
* @param conf the Configuration to use (e.g., from ToolRunner).
* @param opts the SqoopOptions which control the tool's parameters.
*/
public Sqoop(SqoopTool tool, Configuration conf, SqoopOptions opts) {
if (null != conf) {
setConf(conf);
}
this.options = opts;
this.options.setConf(getConf());
this.tool = tool;
}
/**
* @return the SqoopOptions used in this Sqoop instance.
*/
public SqoopOptions getOptions() {
return this.options;
}
/**
* @return the SqoopTool used in this Sqoop instance.
*/
public SqoopTool getTool() {
return this.tool;
}
@Override
/**
* Actual main entry-point for the program
*/
public int run(String [] args) {
if (options.getConf() == null) {
// Configuration wasn't initialized until after the ToolRunner
// got us to this point. ToolRunner gave Sqoop itself a Conf
// though.
options.setConf(getConf());
}
try {
options = tool.parseArguments(args, null, options, false);
tool.appendArgs(this.childPrgmArgs);
tool.validateOptions(options);
} catch (Exception e) {
// Couldn't parse arguments.
// Log the stack trace for this exception
LOG.debug(e.getMessage(), e);
// Print exception message.
System.err.println(e.getMessage());
// Print the tool usage message and exit.
ToolOptions toolOpts = new ToolOptions();
tool.configureOptions(toolOpts);
tool.printHelp(toolOpts);
return 1; // Exit on exception here.
}
return tool.run(options);
}
/**
* SqoopTools sometimes pass arguments to a child program (e.g., mysqldump).
* Users can specify additional args to these programs by preceeding the
* additional arguments with a standalone '--'; but
* ToolRunner/GenericOptionsParser will cull out this argument. We remove
* the child-program arguments in advance, and store them to be readded
* later.
* @param argv the argv in to the SqoopTool
* @return the argv with a "--" and any subsequent arguments removed.
*/
private String [] stashChildPrgmArgs(String [] argv) {
for (int i = 0; i < argv.length; i++) {
if ("--".equals(argv[i])) {
this.childPrgmArgs = Arrays.copyOfRange(argv, i, argv.length);
return Arrays.copyOfRange(argv, 0, i);
}
}
// Didn't find child-program arguments.
return argv;
}
/**
* Given a Sqoop object and a set of arguments to deliver to
* its embedded SqoopTool, run the tool, wrapping the call to
* ToolRunner.
* This entry-point is preferred to ToolRunner.run() because
* it has a chance to stash child program arguments before
* GenericOptionsParser would remove them.
*/
public static int runSqoop(Sqoop sqoop, String [] args) {
try {
String [] toolArgs = sqoop.stashChildPrgmArgs(args);
return ToolRunner.run(sqoop.getConf(), sqoop, toolArgs);
} catch (Exception e) {
LOG.error("Got exception running Sqoop: " + e.toString());
e.printStackTrace();
if (System.getProperty(SQOOP_RETHROW_PROPERTY) != null) {
throw new RuntimeException(e);
}
return 1;
}
}
/**
* Entry-point that parses the correct SqoopTool to use from the args,
* but does not call System.exit() as main() will.
*/
public static int runTool(String [] args, Configuration conf) {
// Expand the options
String[] expandedArgs = null;
try {
expandedArgs = OptionsFileUtil.expandArguments(args);
} catch (Exception ex) {
LOG.error("Error while expanding arguments", ex);
System.err.println(ex.getMessage());
System.err.println("Try 'sqoop help' for usage.");
return 1;
}
String toolName = expandedArgs[0];
Configuration pluginConf = SqoopTool.loadPlugins(conf);
SqoopTool tool = SqoopTool.getTool(toolName);
if (null == tool) {
System.err.println("No such sqoop tool: " + toolName
+ ". See 'sqoop help'.");
return 1;
}
Sqoop sqoop = new Sqoop(tool, pluginConf);
return runSqoop(sqoop,
Arrays.copyOfRange(expandedArgs, 1, expandedArgs.length));
}
/**
* Entry-point that parses the correct SqoopTool to use from the args,
* but does not call System.exit() as main() will.
*/
public static int runTool(String [] args) {
return runTool(args, new Configuration());
}
public static void main(String [] args) {
if (args.length == 0) {
System.err.println("Try 'sqoop help' for usage.");
System.exit(1);
}
int ret = runTool(args);
System.exit(ret);
}
}

1961
src/java/org/apache/sqoop/SqoopOptions.java Normal file
View File

File diff suppressed because it is too large Load Diff

2
src/java/org/apache/sqoop/lib/ClobRef.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information

14
src/java/org/apache/sqoop/lib/DelimiterSet.java
View File

@ -17,7 +17,6 @@
*/
package org.apache.sqoop.lib;
/**
* Encapsulates a set of delimiters used to encode a record.
*/
@ -62,6 +61,19 @@ public DelimiterSet(char field, char record, char enclose, char escape,
this.encloseRequired = isEncloseRequired;
}
/**
* Identical to clone() but does not throw spurious exceptions.
* @return a new copy of this same set of delimiters.
*/
public DelimiterSet copy() {
try {
return (DelimiterSet) clone();
} catch (CloneNotSupportedException cnse) {
// Should never happen for DelimiterSet.
return null;
}
}
/**
* Sets the fields-terminated-by character.
*/

6
src/java/org/apache/sqoop/manager/PostgresqlManager.java
View File

@ -150,5 +150,11 @@ protected String getPrimaryKeyQuery(String tableName) {
private String escapeLiteral(String literal) {
return literal.replace("'", "''");
}
@Override
protected String getCurTimestampQuery() {
return "SELECT CURRENT_TIMESTAMP";
}
}

3
src/java/org/apache/sqoop/mapreduce/AvroExportMapper.java
View File

@ -58,7 +58,8 @@ public class AvroExportMapper
private static final String BIG_DECIMAL_TYPE = "java.math.BigDecimal";
public static final String AVRO_COLUMN_TYPES_MAP = "sqoop.avro.column.types.map";
public static final String AVRO_COLUMN_TYPES_MAP =
"sqoop.avro.column.types.map";
private MapWritable columnTypes;
private SqoopRecord recordImpl;

1
src/java/org/apache/sqoop/mapreduce/MySQLDumpInputFormat.java
View File

@ -24,7 +24,6 @@
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import com.cloudera.sqoop.mapreduce.DataDrivenImportJob;
import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
/**

6
src/java/org/apache/sqoop/mapreduce/RawKeyTextOutputFormat.java
View File

@ -32,11 +32,15 @@
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.*;
/** An {@link OutputFormat} that writes plain text files.
/**
* An {@link OutputFormat} that writes plain text files.
* Only writes the key. Does not write any delimiter/newline after the key.
*/
public class RawKeyTextOutputFormat<K, V> extends FileOutputFormat<K, V> {
/**
* RecordWriter to write to plain text files.
*/
public static class RawKeyRecordWriter<K, V> extends RecordWriter<K, V> {
private static final String UTF8 = "UTF-8";

1
src/java/org/apache/sqoop/mapreduce/db/DBConfiguration.java
View File

@ -25,7 +25,6 @@
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import com.cloudera.sqoop.mapreduce.db.DBInputFormat.NullDBWritable;
import com.cloudera.sqoop.mapreduce.db.DBOutputFormat;
/**
* A container for configuration property names for jobs with DB input/output.

3
src/java/org/apache/sqoop/mapreduce/db/FloatSplitter.java
View File

@ -31,6 +31,9 @@
import com.cloudera.sqoop.mapreduce.db.DBSplitter;
import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
/**
* Implement DBSplitter over floating-point values.
*/
public class FloatSplitter implements DBSplitter {
private static final Log LOG = LogFactory.getLog(FloatSplitter.class);

2
src/java/org/apache/sqoop/mapreduce/db/IntegerSplitter.java
View File

@ -113,7 +113,7 @@ public List<InputSplit> split(Configuration conf, ResultSet results,
* [8, 12)
* [12, 18] note the closed interval for the last split.
*/
protected List<Long> split(long numSplits, long minVal, long maxVal)
public List<Long> split(long numSplits, long minVal, long maxVal)
throws SQLException {
List<Long> splits = new ArrayList<Long>();

6
src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
View File

@ -142,7 +142,7 @@ public List<InputSplit> split(Configuration conf, ResultSet results,
return splits;
}
protected List<String> split(int numSplits, String minString,
public List<String> split(int numSplits, String minString,
String maxString, String commonPrefix) throws SQLException {
BigDecimal minVal = stringToBigDecimal(minString);
@ -183,7 +183,7 @@ protected List<String> split(int numSplits, String minString,
* Return a BigDecimal representation of string 'str' suitable for use in a
* numerically-sorting order.
*/
protected BigDecimal stringToBigDecimal(String str) {
public BigDecimal stringToBigDecimal(String str) {
// Start with 1/65536 to compute the first digit.
BigDecimal curPlace = ONE_PLACE;
BigDecimal result = BigDecimal.ZERO;
@ -208,7 +208,7 @@ protected BigDecimal stringToBigDecimal(String str) {
* Convert that back into a char and create a string out of these until we
* have no data left.
*/
protected String bigDecimalToString(BigDecimal bd) {
public String bigDecimalToString(BigDecimal bd) {
BigDecimal cur = bd.stripTrailingZeros();
StringBuilder sb = new StringBuilder();

2
src/java/org/apache/sqoop/metastore/hsqldb/AutoHsqldbStorage.java
View File

@ -32,7 +32,7 @@
* JobStorage implementation that auto-configures an HSQLDB
* local-file-based instance to hold jobs.
*/
public class AutoHsqldbStorage
public class AutoHsqldbStorage
extends com.cloudera.sqoop.metastore.hsqldb.HsqldbJobStorage {
public static final Log LOG = LogFactory.getLog(

2
src/perftest/ExportStressTest.java
View File

@ -1,6 +1,4 @@
/**
* Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information

Some files were not shown because too many files have changed in this diff Show More