+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager; + +import com.cloudera.sqoop.SqoopOptions; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + + +public enum SupportedManagers { + MYSQL("jdbc:mysql:", true), POSTGRES("jdbc:postgresql:", true), HSQLDB("jdbc:hsqldb:", false), ORACLE("jdbc:oracle:", true), SQLSERVER("jdbc:sqlserver:", false), + JTDS_SQLSERVER("jdbc:jtds:sqlserver:", false), DB2("jdbc:db2:", false), NETEZZA("jdbc:netezza:", true), CUBRID("jdbc:cubrid:", false); + + private final String schemePrefix; + + private final boolean hasDirectConnector; + private static final Log LOG + = LogFactory.getLog(SupportedManagers.class); + + SupportedManagers(String schemePrefix, boolean hasDirectConnector) { + this.schemePrefix = schemePrefix; + this.hasDirectConnector = hasDirectConnector; + } + + public String getSchemePrefix() { + return schemePrefix; + } + + public boolean hasDirectConnector() { + return hasDirectConnector; + } + + public boolean isTheManagerTypeOf(SqoopOptions options) { + return (extractScheme(options)).startsWith(getSchemePrefix()); + } + + public static SupportedManagers createFrom(SqoopOptions options) { + String scheme = extractScheme(options); + for (SupportedManagers m : values()) { + if (scheme.startsWith(m.getSchemePrefix())) { + return m; + } + } + return null; + } + + static String extractScheme(SqoopOptions options) { + String connectStr = options.getConnectString(); + + // java.net.URL follows RFC-2396 literally, which does not allow a ':' + // character in the scheme component (section 3.1). JDBC connect strings, + // however, commonly have a multi-scheme addressing system. e.g., + // jdbc:mysql://...; so we cannot parse the scheme component via URL + // objects. Instead, attempt to pull out the scheme as best as we can. + + // First, see if this is of the form [scheme://hostname-and-etc..] + int schemeStopIdx = connectStr.indexOf("//"); + if (-1 == schemeStopIdx) { + // If no hostname start marker ("//"), then look for the right-most ':' + // character. + schemeStopIdx = connectStr.lastIndexOf(':'); + if (-1 == schemeStopIdx) { + // Warn that this is nonstandard. But we should be as permissive + // as possible here and let the ConnectionManagers themselves throw + // out the connect string if it doesn't make sense to them. + LOG.warn("Could not determine scheme component of connect string"); + + // Use the whole string. + schemeStopIdx = connectStr.length(); + } + } + return connectStr.substring(0, schemeStopIdx); + } +} diff --git a/src/java/org/apache/sqoop/tool/ExportTool.java b/src/java/org/apache/sqoop/tool/ExportTool.java index 15ecddf1..bd51c30c 100644 --- a/src/java/org/apache/sqoop/tool/ExportTool.java +++ b/src/java/org/apache/sqoop/tool/ExportTool.java @@ -387,8 +387,14 @@ public void validateOptions(SqoopOptions options) validateCommonOptions(options); validateCodeGenOptions(options); validateHCatalogOptions(options); + vaildateDirectExportOptions(options); } + void vaildateDirectExportOptions(SqoopOptions options) throws InvalidOptionsException { + if (options.isDirect()) { + validateHasDirectConnectorOption(options); + } + } private void applyNewUpdateOptions(CommandLine in, SqoopOptions out) throws InvalidOptionsException { if (in.hasOption(UPDATE_MODE_ARG)) { diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java index ff7b822c..d3f8b935 100644 --- a/src/java/org/apache/sqoop/tool/ImportTool.java +++ b/src/java/org/apache/sqoop/tool/ImportTool.java @@ -53,6 +53,9 @@ import com.cloudera.sqoop.metastore.JobStorageFactory; import com.cloudera.sqoop.util.AppendUtils; import com.cloudera.sqoop.util.ImportException; +import org.apache.sqoop.manager.SupportedManagers; + +import static org.apache.sqoop.manager.SupportedManagers.MYSQL; /** * Tool that performs database imports to HDFS. @@ -1018,90 +1021,103 @@ public void applyOptions(CommandLine in, SqoopOptions out) * @param options the configured SqoopOptions to check */ protected void validateImportOptions(SqoopOptions options) - throws InvalidOptionsException { - if (!allTables && options.getTableName() == null - && options.getSqlQuery() == null) { - throw new InvalidOptionsException( - "--table or --" + SQL_QUERY_ARG + " is required for import. " - + "(Or use sqoop import-all-tables.)" - + HELP_STR); - } else if (options.getExistingJarName() != null - && options.getClassName() == null) { - throw new InvalidOptionsException("Jar specified with --jar-file, but no " - + "class specified with --class-name." + HELP_STR); - } else if (options.getTargetDir() != null - && options.getWarehouseDir() != null) { - throw new InvalidOptionsException( - "--target-dir with --warehouse-dir are incompatible options." - + HELP_STR); - } else if (options.getTableName() != null - && options.getSqlQuery() != null) { - throw new InvalidOptionsException( - "Cannot specify --" + SQL_QUERY_ARG + " and --table together." - + HELP_STR); - } else if (options.getSqlQuery() != null - && options.getTargetDir() == null - && options.getHBaseTable() == null - && options.getHCatTableName() == null - && options.getAccumuloTable() == null) { - throw new InvalidOptionsException( - "Must specify destination with --target-dir. " - + HELP_STR); - } else if (options.getSqlQuery() != null && options.doHiveImport() - && options.getHiveTableName() == null) { - throw new InvalidOptionsException( - "When importing a query to Hive, you must specify --" - + HIVE_TABLE_ARG + "." + HELP_STR); - } else if (options.getSqlQuery() != null && options.getNumMappers() > 1 - && options.getSplitByCol() == null) { - throw new InvalidOptionsException( - "When importing query results in parallel, you must specify --" - + SPLIT_BY_ARG + "." + HELP_STR); - } else if (options.isDirect() - && options.getFileLayout() != SqoopOptions.FileLayout.TextFile - && options.getConnectString().contains("jdbc:mysql://")) { - throw new InvalidOptionsException( - "MySQL direct import currently supports only text output format. " - + "Parameters --as-sequencefile --as-avrodatafile and --as-parquetfile are not " - + "supported with --direct params in MySQL case."); - } else if (options.isDirect() - && options.doHiveDropDelims()) { - throw new InvalidOptionsException( - "Direct import currently do not support dropping hive delimiters," - + " please remove parameter --hive-drop-import-delims."); - } else if (allTables && options.isValidationEnabled()) { - throw new InvalidOptionsException("Validation is not supported for " - + "all tables but single table only."); - } else if (options.getSqlQuery() != null && options.isValidationEnabled()) { - throw new InvalidOptionsException("Validation is not supported for " - + "free from query but single table only."); - } else if (options.getWhereClause() != null - && options.isValidationEnabled()) { - throw new InvalidOptionsException("Validation is not supported for " - + "where clause but single table only."); - } else if (options.getIncrementalMode() - != SqoopOptions.IncrementalMode.None && options.isValidationEnabled()) { - throw new InvalidOptionsException("Validation is not supported for " - + "incremental imports but single table only."); - } else if ((options.getTargetDir() != null - || options.getWarehouseDir() != null) - && options.getHCatTableName() != null) { - throw new InvalidOptionsException("--hcatalog-table cannot be used " - + " --warehouse-dir or --target-dir options"); - } else if (options.isDeleteMode() && options.isAppendMode()) { - throw new InvalidOptionsException("--append and --delete-target-dir can" - + " not be used together."); - } else if (options.isDeleteMode() && options.getIncrementalMode() - != SqoopOptions.IncrementalMode.None) { - throw new InvalidOptionsException("--delete-target-dir can not be used" - + " with incremental imports."); - } else if (options.getAutoResetToOneMapper() - && (options.getSplitByCol() != null)) { - throw new InvalidOptionsException("--autoreset-to-one-mapper and" - + " --split-by cannot be used together."); - } - } + throws InvalidOptionsException { + if (!allTables && options.getTableName() == null + && options.getSqlQuery() == null) { + throw new InvalidOptionsException( + "--table or --" + SQL_QUERY_ARG + " is required for import. " + + "(Or use sqoop import-all-tables.)" + + HELP_STR); + } else if (options.getExistingJarName() != null + && options.getClassName() == null) { + throw new InvalidOptionsException("Jar specified with --jar-file, but no " + + "class specified with --class-name." + HELP_STR); + } else if (options.getTargetDir() != null + && options.getWarehouseDir() != null) { + throw new InvalidOptionsException( + "--target-dir with --warehouse-dir are incompatible options." + + HELP_STR); + } else if (options.getTableName() != null + && options.getSqlQuery() != null) { + throw new InvalidOptionsException( + "Cannot specify --" + SQL_QUERY_ARG + " and --table together." + + HELP_STR); + } else if (options.getSqlQuery() != null + && options.getTargetDir() == null + && options.getHBaseTable() == null + && options.getHCatTableName() == null + && options.getAccumuloTable() == null) { + throw new InvalidOptionsException( + "Must specify destination with --target-dir. " + + HELP_STR); + } else if (options.getSqlQuery() != null && options.doHiveImport() + && options.getHiveTableName() == null) { + throw new InvalidOptionsException( + "When importing a query to Hive, you must specify --" + + HIVE_TABLE_ARG + "." + HELP_STR); + } else if (options.getSqlQuery() != null && options.getNumMappers() > 1 + && options.getSplitByCol() == null) { + throw new InvalidOptionsException( + "When importing query results in parallel, you must specify --" + + SPLIT_BY_ARG + "." + HELP_STR); + } else if (options.isDirect()) { + validateDirectImportOptions(options); + } else if (allTables && options.isValidationEnabled()) { + throw new InvalidOptionsException("Validation is not supported for " + + "all tables but single table only."); + } else if (options.getSqlQuery() != null && options.isValidationEnabled()) { + throw new InvalidOptionsException("Validation is not supported for " + + "free from query but single table only."); + } else if (options.getWhereClause() != null + && options.isValidationEnabled()) { + throw new InvalidOptionsException("Validation is not supported for " + + "where clause but single table only."); + } else if (options.getIncrementalMode() + != SqoopOptions.IncrementalMode.None && options.isValidationEnabled()) { + throw new InvalidOptionsException("Validation is not supported for " + + "incremental imports but single table only."); + } else if ((options.getTargetDir() != null + || options.getWarehouseDir() != null) + && options.getHCatTableName() != null) { + throw new InvalidOptionsException("--hcatalog-table cannot be used " + + " --warehouse-dir or --target-dir options"); + } else if (options.isDeleteMode() && options.isAppendMode()) { + throw new InvalidOptionsException("--append and --delete-target-dir can" + + " not be used together."); + } else if (options.isDeleteMode() && options.getIncrementalMode() + != SqoopOptions.IncrementalMode.None) { + throw new InvalidOptionsException("--delete-target-dir can not be used" + + " with incremental imports."); + } else if (options.getAutoResetToOneMapper() + && (options.getSplitByCol() != null)) { + throw new InvalidOptionsException("--autoreset-to-one-mapper and" + + " --split-by cannot be used together."); + } + } + void validateDirectImportOptions(SqoopOptions options) throws InvalidOptionsException { + validateDirectMysqlOptions(options); + validateDirectDropHiveDelimOption(options); + validateHasDirectConnectorOption(options); + } + + void validateDirectDropHiveDelimOption(SqoopOptions options) throws InvalidOptionsException { + if (options.doHiveDropDelims()) { + throw new InvalidOptionsException( + "Direct import currently do not support dropping hive delimiters," + + " please remove parameter --hive-drop-import-delims."); + } + } + + void validateDirectMysqlOptions(SqoopOptions options) throws InvalidOptionsException { + if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile + && MYSQL.isTheManagerTypeOf(options)) { + throw new InvalidOptionsException( + "MySQL direct import currently supports only text output format. " + + "Parameters --as-sequencefile --as-avrodatafile and --as-parquetfile are not " + + "supported with --direct params in MySQL case."); + } + } /** * Validate the incremental import options. */ diff --git a/src/test/com/cloudera/sqoop/TestDirectImport.java b/src/test/com/cloudera/sqoop/TestDirectImport.java new file mode 100644 index 00000000..f48c112c --- /dev/null +++ b/src/test/com/cloudera/sqoop/TestDirectImport.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.sqoop;
+
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.testutil.HsqldbTestServer;
+import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import junit.framework.JUnit4TestAdapter;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+@RunWith(value = org.junit.runners.JUnit4.class)
+public class TestDirectImport extends ImportJobTestCase {
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ protected String[] getArgv(boolean includeHadoopFlags, String[] colNames, boolean isDirect) {
+ String columnsString = "";
+ for (String col : colNames) {
+ columnsString += col + ",";
+ }
+
+ ArrayList
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class TestExportToolValidateOptions {
+ ExportTool exportTool = new ExportTool();
+
+
+ @Test
+ public void givenDirectImportHasDirectConnectorValidationPasses() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.NETEZZA);
+ exportTool.vaildateDirectExportOptions(options);
+ }
+
+ @Test(expected = org.apache.sqoop.SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportNoDirectConnectorValidationThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.HSQLDB);
+ exportTool.vaildateDirectExportOptions(options);
+ }
+
+ @Test
+ public void givenNoDirectOptionWhenNoDirectConnectorAvailableValidationPasses() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubNotDirectOptions(SupportedManagers.HSQLDB);
+ exportTool.vaildateDirectExportOptions(options);
+ }
+
+ private SqoopOptions stubDirectOptions(SupportedManagers supportedManagers) {
+ return stubOptions(supportedManagers, true);
+ }
+
+ private SqoopOptions stubNotDirectOptions(SupportedManagers supportedManagers) {
+ return stubOptions(supportedManagers, false);
+ }
+
+ private SqoopOptions stubOptions(SupportedManagers supportedManagers, boolean isDirect) {
+ SqoopOptions options = mock(SqoopOptions.class);
+ when(options.getConnectString()).thenReturn(supportedManagers.getSchemePrefix() + "//localhost");
+ when(options.isDirect()).thenReturn(isDirect);
+ when(options.getConf()).thenReturn(mock(Configuration.class));
+ return options;
+ }
+}
diff --git a/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java b/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java
new file mode 100644
index 00000000..acf4fcf5
--- /dev/null
+++ b/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import com.cloudera.sqoop.SqoopOptions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.manager.SupportedManagers;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class TestValidateImportOptions {
+
+ ImportTool importTool = new ImportTool();
+
+ private static final String mysqlConnectionString = "jdbc:mysql://" + RandomStringUtils.random(5);
+
+
+ @Test
+ public void givenDirectImportMysqlTextFileValidationPasses() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = mock(SqoopOptions.class);
+ when(options.getFileLayout()).thenReturn(SqoopOptions.FileLayout.TextFile);
+ when(options.getConnectString()).thenReturn(mysqlConnectionString);
+ importTool.validateDirectMysqlOptions(options);
+ verify(options, times(1)).getFileLayout();
+ verifyNoMoreInteractions(options);
+ }
+
+
+ @Test(expected = org.apache.sqoop.SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportMysqlSequenceFileValidationThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = mock(SqoopOptions.class);
+ when(options.getFileLayout()).thenReturn(SqoopOptions.FileLayout.SequenceFile);
+ when(options.getConnectString()).thenReturn(mysqlConnectionString);
+ importTool.validateDirectMysqlOptions(options);
+ verify(options, times(1)).getFileLayout();
+ verify(options, times(1)).getConnectString();
+ verifyNoMoreInteractions(options);
+ }
+
+ @Test(expected = org.apache.sqoop.SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportMysqlParquetFileValidationThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = mock(SqoopOptions.class);
+ when(options.getFileLayout()).thenReturn(SqoopOptions.FileLayout.ParquetFile);
+ when(options.getConnectString()).thenReturn(mysqlConnectionString);
+ importTool.validateDirectMysqlOptions(options);
+ verify(options, times(1)).getFileLayout();
+ verify(options, times(1)).getConnectString();
+ verifyNoMoreInteractions(options);
+ }
+ @Test(expected = org.apache.sqoop.SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportMysqlAvroDataFileValidationThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = mock(SqoopOptions.class);
+ when(options.getFileLayout()).thenReturn(SqoopOptions.FileLayout.AvroDataFile);
+ when(options.getConnectString()).thenReturn(mysqlConnectionString);
+ importTool.validateDirectMysqlOptions(options);
+ verify(options, times(1)).getFileLayout();
+ verify(options, times(1)).getConnectString();
+ verifyNoMoreInteractions(options);
+ }
+
+ @Test(expected = org.apache.sqoop.SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportHiveDropDelimValidationThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = mock(SqoopOptions.class);
+ when(options.doHiveDropDelims()).thenReturn(true);
+ importTool.validateDirectDropHiveDelimOption(options);
+ verify(options, times(1)).doHiveDropDelims();
+ verifyNoMoreInteractions(options);
+ }
+
+ @Test
+ public void givenDirectImportHasDirectConnectorValidationPasses() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.NETEZZA);
+ importTool.validateDirectImportOptions(options);
+ }
+
+ @Test(expected = org.apache.sqoop.SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportNoDirectConnectorValidationThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.HSQLDB);
+ importTool.validateDirectImportOptions(options);
+ }
+
+ @Test
+ public void givenNoDirectOptionWhenNoDirectConnectorAvailableValidationPasses() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubNotDirectOptions(SupportedManagers.HSQLDB);
+ importTool.validateDirectImportOptions(options);
+ }
+
+ private SqoopOptions stubDirectOptions(SupportedManagers supportedManagers) {
+ return stubOptions(supportedManagers, true);
+ }
+
+ private SqoopOptions stubNotDirectOptions(SupportedManagers supportedManagers) {
+ return stubOptions(supportedManagers, false);
+ }
+
+ private SqoopOptions stubOptions(SupportedManagers supportedManagers, boolean isDirect) {
+ SqoopOptions options = mock(SqoopOptions.class);
+ when(options.getConnectString()).thenReturn(supportedManagers.getSchemePrefix() + "//localhost");
+ when(options.isDirect()).thenReturn(isDirect);
+ when(options.getConf()).thenReturn(mock(Configuration.class));
+ return options;
+ }
+
+ //TODO create tests for all old validations as well
+}