5
0
mirror of https://github.com/apache/sqoop.git synced 2025-05-04 09:59:14 +08:00

SQOOP-1519: Enable HCat/HBase/Accumulo operations with OraOop connection manager

This commit is contained in:
Venkat Ranganathan 2014-09-15 14:17:40 -07:00 committed by Abraham Elmahrek
parent 57336d7f53
commit ad53e4e334
5 changed files with 70 additions and 10 deletions

View File

@ -794,5 +794,25 @@ public boolean isCharColumn(int columnType) {
public boolean isDirectModeHCatSupported() { public boolean isDirectModeHCatSupported() {
return false; return false;
} }
/**
* Determine if HBase operations from direct mode of the connector is
* allowed. By default direct mode is not compatible with HBase
* @return Whether direct mode is allowed.
*/
public boolean isDirectModeHBaseSupported() {
return false;
}
/**
* Determine if Accumulo operations from direct mode of the connector is
* allowed. By default direct mode is not compatible with HBase
* @return Whether direct mode is allowed.
*/
public boolean isDirectModeAccumuloSupported() {
return false;
}
} }

View File

@ -631,5 +631,31 @@ private void explainWhyExportClassCannotBeLoaded(NoClassDefFoundError ex,
.getJavaClassPath()); .getJavaClassPath());
LOG.fatal(msg, ex); LOG.fatal(msg, ex);
} }
/**
* Determine if HCat integration from direct mode of the connector is
* allowed. By default direct mode is not compatible with HCat
* @return Whether direct mode is allowed.
*/
@Override
public boolean isDirectModeHCatSupported() {
return true;
}
/**
* Determine if HBase operations from direct mode of the connector is
* allowed. By default direct mode is not compatible with HBase
* @return Whether direct mode is allowed.
*/
public boolean isDirectModeHBaseSupported() {
return true;
}
/**
* Determine if Accumulo operations from direct mode of the connector is
* allowed. By default direct mode is not compatible with HBase
* @return Whether direct mode is allowed.
*/
public boolean isDirectModeAccumuloSupported() {
return true;
}
} }

View File

@ -39,7 +39,9 @@
import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities; import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities;
import org.apache.sqoop.util.LoggingUtils; import org.apache.sqoop.util.LoggingUtils;
import org.apache.sqoop.util.PerfCounters; import org.apache.sqoop.util.PerfCounters;
import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
import com.cloudera.sqoop.config.ConfigurationHelper; import com.cloudera.sqoop.config.ConfigurationHelper;
import com.cloudera.sqoop.lib.SqoopRecord; import com.cloudera.sqoop.lib.SqoopRecord;
import com.cloudera.sqoop.manager.ConnManager; import com.cloudera.sqoop.manager.ConnManager;
@ -47,6 +49,7 @@
import com.cloudera.sqoop.orm.TableClassName; import com.cloudera.sqoop.orm.TableClassName;
import com.cloudera.sqoop.mapreduce.JobBase; import com.cloudera.sqoop.mapreduce.JobBase;
import com.cloudera.sqoop.util.ExportException; import com.cloudera.sqoop.util.ExportException;
import org.apache.sqoop.validation.*; import org.apache.sqoop.validation.*;
/** /**
@ -327,7 +330,16 @@ public void runExport() throws ExportException, IOException {
+ context.getConnManager().getClass().getName() + context.getConnManager().getClass().getName()
+ ". Please remove the parameter --direct"); + ". Please remove the parameter --direct");
} }
if (options.getAccumuloTable() != null && options.isDirect()
&& !cmgr.isDirectModeAccumuloSupported()) {
throw new IOException("Direct mode is incompatible with "
+ "Accumulo. Please remove the parameter --direct");
}
if (options.getHBaseTable() != null && options.isDirect()
&& !cmgr.isDirectModeHBaseSupported()) {
throw new IOException("Direct mode is incompatible with "
+ "HBase. Please remove the parameter --direct");
}
if (stagingTableName != null) { // user has specified the staging table if (stagingTableName != null) { // user has specified the staging table
if (cmgr.supportsStagingForExport()) { if (cmgr.supportsStagingForExport()) {
LOG.info("Data will be staged in the table: " + stagingTableName); LOG.info("Data will be staged in the table: " + stagingTableName);

View File

@ -223,7 +223,16 @@ public void runImport(String tableName, String ormJarFile, String splitByCol,
+ context.getConnManager().getClass().getName() + context.getConnManager().getClass().getName()
+ ". Please remove the parameter --direct"); + ". Please remove the parameter --direct");
} }
if (options.getAccumuloTable() != null && options.isDirect()
&& !getContext().getConnManager().isDirectModeAccumuloSupported()) {
throw new IOException("Direct mode is incompatible with "
+ "Accumulo. Please remove the parameter --direct");
}
if (options.getHBaseTable() != null && options.isDirect()
&& !getContext().getConnManager().isDirectModeHBaseSupported()) {
throw new IOException("Direct mode is incompatible with "
+ "HBase. Please remove the parameter --direct");
}
if (null != tableName) { if (null != tableName) {
LOG.info("Beginning import of " + tableName); LOG.info("Beginning import of " + tableName);
} else { } else {

View File

@ -1425,10 +1425,7 @@ protected void validateAccumuloOptions(SqoopOptions options)
"Both --accumulo-table and --accumulo-column-family must be set." "Both --accumulo-table and --accumulo-column-family must be set."
+ HELP_STR); + HELP_STR);
} }
if (options.getAccumuloTable() != null && options.isDirect()) {
throw new InvalidOptionsException("Direct import is incompatible with "
+ "Accumulo. Please remove parameter --direct");
}
if (options.getAccumuloTable() != null if (options.getAccumuloTable() != null
&& options.getHBaseTable() != null) { && options.getHBaseTable() != null) {
throw new InvalidOptionsException("HBase import is incompatible with " throw new InvalidOptionsException("HBase import is incompatible with "
@ -1576,10 +1573,6 @@ protected void validateHBaseOptions(SqoopOptions options)
"Both --hbase-table and --column-family must be set together." "Both --hbase-table and --column-family must be set together."
+ HELP_STR); + HELP_STR);
} }
if (options.getHBaseTable() != null && options.isDirect()) {
throw new InvalidOptionsException("Direct import is incompatible with "
+ "HBase. Please remove parameter --direct");
}
if (options.isBulkLoadEnabled() && options.getHBaseTable() == null) { if (options.isBulkLoadEnabled() && options.getHBaseTable() == null) {
String validationMessage = String.format("Can't run import with %s " String validationMessage = String.format("Can't run import with %s "