mirror of
https://github.com/apache/sqoop.git
synced 2025-05-02 08:11:21 +08:00
7e0ccb4008
Added FieldMappable and FieldMapProcessor interfaces. Added ProcessingException class. Added NullOutputCommitter class. SqoopRecord now has delegate() method which calls a FieldMapProcessor. ClassWriter now generates getFieldMap() method for SqoopRecords. Added HBasePutProcessor to transform SqoopRecords into Put commands, implementing FieldMapProcessor. Added PutTransformer interface class and ToStringPutTransformer implementation. Added DelegatingOutputFormat that uses a FieldMapProcessor. Added HBase deps to build.xml via hbase.home property. Added HBase, ZooKeeper to the dependency net added by configure-sqoop. Added HBaseImportJob, HBaseImportMapper. ImportJobBase now has jobSetup() step executed just before job submission. ImportJobContext now holds a reference to the ConnManager. DataDrivenImportJob retrieves ConnManager from ImportJobContext, it no longer creates a new one. Added HBase table import configuration parameters to SqoopOptions, ImportTool. SqlManager.importQuery() needs to set ConnManager in ImportJobContext. Added HBase import user documentation. Described PutTransformer API in developer docs. Added HBase unit tests. Added ANT_ARGUMENTS env variable to Hudson test scripts to allow freeform parameters. Added HBASE_HOME and ZOOKEEPER_HOME variables to hudson scripts. From: Aaron Kimball <aaron@cloudera.com> git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149935 13f79535-47bb-0310-9956-ffa450edef68
122 lines
3.4 KiB
Bash
Executable File
122 lines
3.4 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Licensed to Cloudera, Inc. under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# This is sourced in by bin/sqoop to set environment variables prior to
|
|
# invoking Hadoop.
|
|
|
|
bin="$1"
|
|
|
|
if [ -z "${bin}" ]; then
|
|
bin=`dirname $0`
|
|
bin=`cd ${bin} && pwd`
|
|
fi
|
|
|
|
if [ -z "$SQOOP_HOME" ]; then
|
|
export SQOOP_HOME=${bin}/..
|
|
fi
|
|
|
|
# Find paths to our dependency systems. If they are unset, use CDH defaults.
|
|
|
|
if [ -z "${HADOOP_HOME}" ]; then
|
|
HADOOP_HOME=/usr/lib/hadoop
|
|
fi
|
|
if [ -z "${HBASE_HOME}" ]; then
|
|
HBASE_HOME=/usr/lib/hbase
|
|
fi
|
|
if [ -z "${ZOOKEEPER_HOME}" ]; then
|
|
ZOOKEEPER_HOME=/usr/lib/zookeeper
|
|
fi
|
|
|
|
# Check: If we can't find our dependencies, give up here.
|
|
if [ ! -d "${HADOOP_HOME}" ]; then
|
|
echo "Error: $HADOOP_HOME does not exist!"
|
|
echo "Please set $$HADOOP_HOME to the root of your Hadoop installation."
|
|
exit 1
|
|
fi
|
|
if [ ! -d "${HBASE_HOME}" ]; then
|
|
echo "Error: $HBASE_HOME does not exist!"
|
|
echo "Please set $$HBASE_HOME to the root of your HBase installation."
|
|
exit 1
|
|
fi
|
|
if [ ! -d "${ZOOKEEPER_HOME}" ]; then
|
|
echo "Error: $ZOOKEEPER_HOME does not exist!"
|
|
echo "Please set $$ZOOKEEPER_HOME to the root of your ZooKeeper installation."
|
|
exit 1
|
|
fi
|
|
|
|
|
|
# Where to find the main Sqoop jar
|
|
SQOOP_JAR_DIR=$SQOOP_HOME
|
|
|
|
# Where to find the shim jars.
|
|
SQOOP_SHIM_DIR=$SQOOP_HOME/shims
|
|
|
|
# If there's a "build" subdir, override with this, so we use
|
|
# the newly-compiled copy.
|
|
if [ -d "$SQOOP_JAR_DIR/build" ]; then
|
|
SQOOP_JAR_DIR="${SQOOP_JAR_DIR}/build"
|
|
|
|
if [ -d "$SQOOP_JAR_DIR/shims" ]; then
|
|
SQOOP_SHIM_DIR="$SQOOP_JAR_DIR/shims"
|
|
fi
|
|
fi
|
|
|
|
function add_to_classpath() {
|
|
dir=$1
|
|
for f in $dir/*.jar; do
|
|
SQOOP_CLASSPATH=${SQOOP_CLASSPATH}:$f;
|
|
done
|
|
|
|
export SQOOP_CLASSPATH
|
|
}
|
|
|
|
# Add sqoop dependencies to classpath.
|
|
SQOOP_CLASSPATH=""
|
|
if [ -d "$SQOOP_HOME/lib" ]; then
|
|
add_to_classpath $SQOOP_HOME/lib
|
|
fi
|
|
|
|
# Add HBase to dependency list
|
|
add_to_classpath $HBASE_HOME
|
|
add_to_classpath $HBASE_HOME/lib
|
|
|
|
HBASE_CONF_DIR=${HBASE_CONF_DIR:-${HBASE_HOME}/conf}
|
|
SQOOP_CLASSPATH=${HBASE_CONF_DIR}:${SQOOP_CLASSPATH}
|
|
|
|
add_to_classpath $ZOOKEEPER_HOME
|
|
add_to_classpath $ZOOKEEPER_HOME/lib
|
|
|
|
SQOOP_CONF_DIR=${SQOOP_CONF_DIR:-${SQOOP_HOME}/conf}
|
|
SQOOP_CLASSPATH=${SQOOP_CONF_DIR}:${SQOOP_CLASSPATH}
|
|
|
|
# If there's a build subdir, use Ivy-retrieved dependencies too.
|
|
if [ -d "$SQOOP_HOME/build/ivy/lib/sqoop" ]; then
|
|
for f in $SQOOP_HOME/build/ivy/lib/sqoop/*/*.jar; do
|
|
SQOOP_CLASSPATH=${SQOOP_CLASSPATH}:$f;
|
|
done
|
|
fi
|
|
|
|
export SQOOP_CLASSPATH
|
|
export SQOOP_JAR_DIR
|
|
export SQOOP_SHIM_DIR
|
|
export SQOOP_JAR=`ls -1 ${SQOOP_JAR_DIR}/sqoop-*.jar | head -n 1`
|
|
export HADOOP_CLASSPATH="${SQOOP_CLASSPATH}:${HADOOP_CLASSPATH}"
|
|
export HADOOP_HOME
|
|
export HBASE_HOME
|
|
export HADOOP_OPTS="-Dsqoop.shim.jar.dir=${SQOOP_SHIM_DIR} ${HADOOP_OPTS}"
|
|
|