diff --git a/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java b/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java index 91a1b8bb..6f32e04b 100644 --- a/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java +++ b/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java @@ -36,13 +36,6 @@ public final class SecurityConstants { public static final String PREFIX_AUTHENTICATION_CONFIG = PREFIX_SECURITY_CONFIG + "authentication."; - /** - * All tls related configuration is prefixed with this: - * org.apache.sqoop.security.tls. - */ - public static final String PREFIX_TLS_CONFIG = - PREFIX_SECURITY_CONFIG + "tls."; - /** * The config specifies the sqoop authentication type (SIMPLE, KERBEROS). * The default type is SIMPLE @@ -164,42 +157,6 @@ public final class SecurityConstants { public static final String SERVER_NAME = PREFIX_AUTHORIZATION_CONFIG + "server_name"; - /** - * The config specifies the whether the http server should use TLS. - * org.apache.sqoop.security.tls.enabled. - */ - public static final String TLS_ENABLED = - PREFIX_TLS_CONFIG + "enabled"; - - /** - * The config specifies the tls protocol version - * org.apache.sqoop.security.tls.protocol. - */ - public static final String TLS_PROTOCOL = - PREFIX_TLS_CONFIG + "protocol"; - - /** - * The config specifies the location of the JKS formatted keystore - * org.apache.sqoop.security.tls.keystore. - */ - public static final String KEYSTORE_LOCATION = - PREFIX_TLS_CONFIG + "keystore"; - - /** - * The config specifies the password of the JKS formatted keystore - * org.apache.sqoop.security.tls.keystorepassword. - */ - public static final String KEYSTORE_PASSWORD = - PREFIX_TLS_CONFIG + "keystore_password"; - - /** - * The config specifies the password for the private key in the JKS formatted - * keystore - * org.apache.sqoop.security.tls.keymanagerpassword. - */ - public static final String KEYMANAGER_PASSWORD = - PREFIX_TLS_CONFIG + "keymanager_password"; - /** * The config specifies the token kind in delegation token. */ diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties index 767d3f2b..620146d5 100755 --- a/dist/src/main/conf/sqoop.properties +++ b/dist/src/main/conf/sqoop.properties @@ -180,15 +180,6 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce #org.apache.sqoop.security.authorization.authentication_provider=org.apache.sqoop.security.authorization.DefaultAuthenticationProvider #org.apache.sqoop.security.authorization.server_name=SqoopServer1 -# -# SSL/TLS configuration -# -#org.apache.sqoop.security.tls.enabled=false -#org.apache.sqoop.security.tls.protocol="TLSv1.2" -#org.apache.sqoop.security.tls.keystore= -#org.apache.sqoop.security.tls.keystore_password= - - # External connectors load path # "/path/to/external/connectors/": Add all the connector JARs in the specified folder # diff --git a/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java b/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java index cb796549..5ea3aa06 100644 --- a/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java +++ b/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java @@ -20,7 +20,6 @@ import org.apache.log4j.Logger; import org.apache.sqoop.docs.generator.plugins.AbstractPlugin; import org.apache.sqoop.docs.generator.plugins.CopySourceToDestination; -import org.apache.sqoop.docs.generator.plugins.GenerateConnectorPages; import org.apache.sqoop.utils.ClassUtils; import java.util.LinkedList; @@ -37,7 +36,6 @@ public class DocPreprocessor { static { plugins = new LinkedList<>(); plugins.add(CopySourceToDestination.class); - plugins.add(GenerateConnectorPages.class); } public static void main(String []args) { diff --git a/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java b/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java deleted file mode 100644 index 119b76d6..00000000 --- a/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.docs.generator.plugins; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; -import org.apache.sqoop.connector.ConnectorHandler; -import org.apache.sqoop.connector.ConnectorManagerUtils; -import org.apache.sqoop.connector.spi.SqoopConnector; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.URL; -import java.util.Collections; -import java.util.List; - -public class GenerateConnectorPages extends AbstractPlugin { - - private static final Logger LOG = Logger.getLogger(GenerateConnectorPages.class); - - @Override - public void run() { - try { - runWithException(); - } catch (IOException e) { - throw new RuntimeException("Can't generate connector documentation", e); - } - } - - public void runWithException() throws IOException { - List connectorUrls = ConnectorManagerUtils.getConnectorConfigs(Collections.emptySet()); - for (URL url : connectorUrls) { - SqoopConnector connector = new ConnectorHandler(url).getSqoopConnector(); - LOG.info("Loaded " + connector.getConnectorName() + " from " + url.toString()); - - String connectorName = connector.getConnectorName(); - - File outputFile = FileUtils.getFile(getDestination(), "user", "connectors", connectorName + ".rst"); - PrintWriter writer = new PrintWriter(outputFile); - - // Writing headline - writer.println(connectorName); - writer.println(StringUtils.repeat("=", connectorName.length())); - - writer.close(); - } - } -} diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java index 60368afc..2c4cb7a1 100644 --- a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java +++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java @@ -19,21 +19,12 @@ package org.apache.sqoop.server; import org.apache.log4j.Logger; -import org.apache.sqoop.common.MapContext; -import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.core.SqoopConfiguration; import org.apache.sqoop.core.SqoopServer; import org.apache.sqoop.filter.SqoopAuthenticationFilter; -import org.apache.sqoop.security.SecurityConstants; -import org.apache.sqoop.server.common.ServerError; import org.apache.sqoop.server.v1.*; -import org.eclipse.jetty.server.HttpConfiguration; -import org.eclipse.jetty.server.HttpConnectionFactory; -import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.ExecutorThreadPool; import org.eclipse.jetty.server.ServerConnector; @@ -67,45 +58,7 @@ public SqoopJettyServer() { webServer = new Server(threadPool); // Connector configs - ServerConnector connector; - - MapContext configurationContext = SqoopConfiguration.getInstance().getContext(); - - if (configurationContext.getBoolean(SecurityConstants.TLS_ENABLED, false)) { - String keyStorePath = configurationContext.getString(SecurityConstants.KEYSTORE_LOCATION); - if (keyStorePath == null) { - throw new SqoopException(ServerError.SERVER_0007); - } - - SslContextFactory sslContextFactory = new SslContextFactory(); - sslContextFactory.setKeyStorePath(keyStorePath); - - String protocol = configurationContext.getString(SecurityConstants.TLS_PROTOCOL); - if (protocol != null && protocol.length() > 0) { - sslContextFactory.setProtocol(protocol.trim()); - } - - String keyStorePassword = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD); - if (keyStorePassword != null && keyStorePassword.length() > 0) { - sslContextFactory.setKeyStorePassword(keyStorePassword); - } - - String keyManagerPassword = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD); - if (keyManagerPassword != null && keyManagerPassword.length() > 0) { - sslContextFactory.setKeyManagerPassword(keyManagerPassword); - } - - HttpConfiguration https = new HttpConfiguration(); - https.addCustomizer(new SecureRequestCustomizer()); - - connector = new ServerConnector(webServer, - new SslConnectionFactory(sslContextFactory, "http/1.1"), - new HttpConnectionFactory(https)); - } else { - connector = new ServerConnector(webServer); - } - - + ServerConnector connector = new ServerConnector(webServer); connector.setPort(sqoopJettyContext.getPort()); webServer.addConnector(connector); webServer.setHandler(createServletContextHandler()); diff --git a/server/src/main/java/org/apache/sqoop/server/common/ServerError.java b/server/src/main/java/org/apache/sqoop/server/common/ServerError.java index a644e9f1..1b021cf2 100644 --- a/server/src/main/java/org/apache/sqoop/server/common/ServerError.java +++ b/server/src/main/java/org/apache/sqoop/server/common/ServerError.java @@ -38,9 +38,6 @@ public enum ServerError implements ErrorCode { /** Entity requested doesn't exist*/ SERVER_0006("Entity requested doesn't exist"), - - /** TLS enabled but keystore location not set*/ - SERVER_0007("TLS enabled but keystore location not set"), ; private final String message; diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java b/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java index b7f11295..36d07125 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java @@ -21,14 +21,12 @@ import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.shell.core.ShellError; import org.apache.sqoop.shell.core.Constants; -import org.apache.sqoop.shell.utils.TableDisplayer; import org.codehaus.groovy.tools.shell.IO; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.text.MessageFormat; -import java.util.List; import java.util.Locale; import java.util.ResourceBundle; @@ -59,13 +57,6 @@ private ShellEnvironment() { private static boolean interactive = false; private static long pollTimeout = DEFAULT_POLL_TIMEOUT; - private static TableDisplayer tableDisplayer = new TableDisplayer(new TableDisplayer.TableDisplayerWriter() { - @Override - public void append(String text) { - print(text); - } - }); - static ResourceBundle resource = ResourceBundle.getBundle(Constants.RESOURCE_NAME, Locale.getDefault()); static SqoopClient client = new SqoopClient(getServerUrl()); static IO io; @@ -235,10 +226,6 @@ public static void print(String format, Object... args) { io.out.printf(format, args); } - public static void displayTable(List headers, List ...columns) { - tableDisplayer.display(headers, columns); - } - // for tests only public static void cleanup() { serverHost = DEFAULT_SERVER_HOST; diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java index c90fe941..d973499d 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java @@ -27,6 +27,7 @@ import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.model.MConnector; import org.apache.sqoop.shell.core.Constants; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import static org.apache.sqoop.shell.ShellEnvironment.*; @@ -82,7 +83,7 @@ private void showSummary() { supportedDirections.add(connector.getSupportedDirections().toString()); } - displayTable(header, uniqueNames, versions, classes, supportedDirections); + TableDisplayer.display(header, uniqueNames, versions, classes, supportedDirections); } private void showConnectors() { diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java index 139e7565..ebbfe828 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java @@ -19,8 +19,12 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.OptionBuilder; +import org.apache.sqoop.common.Direction; +import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; import org.apache.sqoop.shell.core.Constants; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import java.text.DateFormat; @@ -93,7 +97,7 @@ private void showSummary() { availabilities.add(String.valueOf(job.getEnabled())); } - displayTable(header, ids, names, fromConnectors, toConnectors, availabilities); + TableDisplayer.display(header, ids, names, fromConnectors, toConnectors, availabilities); } private void showJobs(String jArg) { diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java index 4976c98f..25bd1dba 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java @@ -19,13 +19,19 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.OptionBuilder; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MLink; import org.apache.sqoop.shell.core.Constants; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import java.text.DateFormat; +import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import static org.apache.sqoop.shell.ShellEnvironment.*; import static org.apache.sqoop.shell.utils.ConfigDisplayer.*; @@ -80,7 +86,7 @@ private void showSummary() { availabilities.add(String.valueOf(link.getEnabled())); } - displayTable(header, names, connectorNames, availabilities); + TableDisplayer.display(header, names, connectorNames, availabilities); } private void showLinks() { diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java index 45abf148..a450aafd 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java @@ -22,13 +22,13 @@ import org.apache.sqoop.model.MPrincipal; import org.apache.sqoop.model.MRole; import org.apache.sqoop.shell.core.Constants; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import java.util.LinkedList; import java.util.List; import static org.apache.sqoop.shell.ShellEnvironment.client; -import static org.apache.sqoop.shell.ShellEnvironment.displayTable; import static org.apache.sqoop.shell.ShellEnvironment.resourceString; @SuppressWarnings("serial") @@ -70,6 +70,6 @@ private void showPrincipals(MRole role) { types.add(principal.getType()); } - displayTable(header, names, types); + TableDisplayer.display(header, names, types); } } diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java index efd4b194..2cf69724 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java @@ -25,13 +25,13 @@ import org.apache.sqoop.model.MResource; import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.core.ShellError; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import java.util.LinkedList; import java.util.List; import static org.apache.sqoop.shell.ShellEnvironment.client; -import static org.apache.sqoop.shell.ShellEnvironment.displayTable; import static org.apache.sqoop.shell.ShellEnvironment.resourceString; @SuppressWarnings("serial") @@ -108,6 +108,6 @@ private void showPrivileges(MPrincipal principal, MResource resource) { withGrant.add(Boolean.toString(privilege.isWith_grant_option())); } - displayTable(header, actions, resourceNames, resourceTypes, withGrant); + TableDisplayer.display(header, actions, resourceNames, resourceTypes, withGrant); } } diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java index 91ceff30..6b619210 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java @@ -24,13 +24,13 @@ import org.apache.sqoop.model.MRole; import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.core.ShellError; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import java.util.LinkedList; import java.util.List; import static org.apache.sqoop.shell.ShellEnvironment.client; -import static org.apache.sqoop.shell.ShellEnvironment.displayTable; import static org.apache.sqoop.shell.ShellEnvironment.resourceString; @SuppressWarnings("serial") @@ -83,6 +83,6 @@ private void showRoles(MPrincipal principal) { names.add(role.getName()); } - displayTable(header, names); + TableDisplayer.display(header, names); } } diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java index c7f371f5..89899134 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java @@ -25,6 +25,7 @@ import org.apache.sqoop.model.MSubmission; import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.utils.SubmissionDisplayer; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import static org.apache.sqoop.shell.ShellEnvironment.*; @@ -90,7 +91,7 @@ private void showSummary(String jArg) { dates.add(submission.getLastUpdateDate().toString()); } - displayTable(header, jnames, eids, status, dates); + TableDisplayer.display(header, jnames, eids, status, dates); } private void showSubmissions(String jArg) { diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java index ba3698ed..51030d03 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java +++ b/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java @@ -22,43 +22,23 @@ import java.util.LinkedList; import java.util.List; +import static org.apache.sqoop.shell.ShellEnvironment.*; + /** * Display table based data */ public class TableDisplayer { - /** - * Interface that this displayer will use to write out formatted text. - */ - public interface TableDisplayerWriter { - /** - * Print out addition formatted text to the output - * - * @param text - */ - void append(String text); - } - - public TableDisplayer(TableDisplayerWriter writer) { - this.writer = writer; - } - - /** - * Writer instance that we should use to write something out - */ - private TableDisplayerWriter writer; - /** * Display given columns in nice table structure to given IO object. * * @param headers List of headers * @param columns Array of columns */ - public void display(List headers, List ...columns) { + public static void display(List headers, List ...columns) { assert headers != null; assert columns != null; assert headers.size() == columns.length; - assert writer != null; // Count of columns int columnCount = headers.size(); @@ -105,7 +85,7 @@ public void display(List headers, List ...columns) { * * @param widths List of widths of each column */ - private void drawLine(List widths) { + private static void drawLine(List widths) { int last = widths.size() - 1; print("+-"); for(int i = 0; i < widths.size(); i++) { @@ -122,7 +102,7 @@ private void drawLine(List widths) { * @param column All column values * @return Maximal */ - private int getMaximalWidth(String header, List column) { + private static int getMaximalWidth(String header, List column) { assert header != null; assert column != null; @@ -143,7 +123,7 @@ private int getMaximalWidth(String header, List column) { * @param columns Array with all column values * @return */ - private int getMaximalRows(List... columns) { + private static int getMaximalRows(List... columns) { int max = 0; for(List column : columns) { @@ -155,12 +135,7 @@ private int getMaximalRows(List... columns) { return max; } - private void print(String text) { - writer.append(text); + private TableDisplayer() { + // Instantiation is prohibited } - - private void println() { - writer.append("\n"); - } - } diff --git a/test/pom.xml b/test/pom.xml index 4bac683d..1e88b34a 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -274,17 +274,17 @@ limitations under the License. - server-properties-test + connector-loading-test test integration-test - src/test/resources/server-properties-tests-suite.xml + src/test/resources/connector-loading-tests-suite.xml - server-properties-tests + connector-loading-tests diff --git a/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java b/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java index bca4412d..83f960b8 100644 --- a/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java +++ b/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java @@ -18,12 +18,31 @@ package org.apache.sqoop.test.kdc; import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; import java.lang.reflect.Constructor; +import java.math.BigInteger; import java.net.URL; +import java.security.GeneralSecurityException; +import java.security.InvalidKeyException; +import java.security.Key; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; import java.security.Principal; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; +import java.security.SecureRandom; +import java.security.SignatureException; +import java.security.cert.Certificate; +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; import java.util.Collection; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -34,16 +53,19 @@ import javax.security.auth.Subject; import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.LoginContext; +import javax.security.auth.x500.X500Principal; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory; +import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; import org.apache.sqoop.client.SqoopClient; import org.apache.sqoop.model.MConnector; import org.apache.sqoop.test.utils.HdfsUtils; -import org.apache.sqoop.test.utils.SecurityUtils; import org.apache.sqoop.test.utils.SqoopUtils; +import org.bouncycastle.x509.X509V1CertificateGenerator; /** * Represents a Minikdc setup. Minikdc should be only used together with @@ -85,17 +107,17 @@ public Configuration prepareHadoopConfiguration(Configuration config) throws Exc config.set("dfs.datanode.kerberos.principal", hadoopPrincipal); config.set("dfs.datanode.keytab.file", hadoopKeytabFile); String sslKeystoresDir = getTemporaryPath() + "/ssl-keystore"; - String sslConfDir = SqoopUtils.getClasspathDir(MiniKdcRunner.class); + String sslConfDir = getClasspathDir(MiniKdcRunner.class); FileUtils.deleteDirectory(new File(sslKeystoresDir)); FileUtils.forceMkdir(new File(sslKeystoresDir)); - SecurityUtils.setupSSLConfig(sslKeystoresDir, sslConfDir, config, false, true); - config.set("dfs.https.server.keystore.resource", SecurityUtils.getSSLConfigFileName("ssl-server")); + setupSSLConfig(sslKeystoresDir, sslConfDir, config, false, true); + config.set("dfs.https.server.keystore.resource", getSSLConfigFileName("ssl-server")); // Configurations used by both NameNode and DataNode config.set("dfs.block.access.token.enable", "true"); config.set("dfs.http.policy", "HTTPS_ONLY"); // Configurations used by DFSClient config.set("dfs.data.transfer.protection", "privacy"); - config.set("dfs.client.https.keystore.resource", SecurityUtils.getSSLConfigFileName("ssl-client")); + config.set("dfs.client.https.keystore.resource", getSSLConfigFileName("ssl-client")); // YARN related configurations config.set("yarn.resourcemanager.principal", hadoopPrincipal); @@ -309,4 +331,219 @@ public T run() throws Exception { } } } + + @SuppressWarnings("rawtypes") + private static String getClasspathDir(Class klass) throws Exception { + String file = klass.getName(); + file = file.replace('.', '/') + ".class"; + URL url = Thread.currentThread().getContextClassLoader().getResource(file); + String baseDir = url.toURI().getPath(); + baseDir = baseDir.substring(0, baseDir.length() - file.length() - 1); + return baseDir; + } + + /** + * Performs complete setup of SSL configuration. This includes keys, certs, + * keystores, truststores, the server SSL configuration file, + * the client SSL configuration file. + * + * @param keystoresDir String directory to save keystores + * @param sslConfDir String directory to save SSL configuration files + * @param conf Configuration + * @param useClientCert boolean true to make the client present a cert in the + * SSL handshake + * @param trustStore boolean true to create truststore, false not to create it + */ + private void setupSSLConfig(String keystoresDir, String sslConfDir, + Configuration conf, boolean useClientCert, boolean trustStore) + throws Exception { + String clientKS = keystoresDir + "/clientKS.jks"; + String clientPassword = "clientP"; + String serverKS = keystoresDir + "/serverKS.jks"; + String serverPassword = "serverP"; + String trustKS = null; + String trustPassword = "trustP"; + + File sslClientConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-client")); + File sslServerConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-server")); + + Map certs = new HashMap(); + + if (useClientCert) { + KeyPair cKP = generateKeyPair("RSA"); + X509Certificate cCert = generateCertificate("CN=localhost, O=client", cKP, 30, "SHA1withRSA"); + createKeyStore(clientKS, clientPassword, "client", cKP.getPrivate(), cCert); + certs.put("client", cCert); + } + + KeyPair sKP = generateKeyPair("RSA"); + X509Certificate sCert = generateCertificate("CN=localhost, O=server", sKP, 30, "SHA1withRSA"); + createKeyStore(serverKS, serverPassword, "server", sKP.getPrivate(), sCert); + certs.put("server", sCert); + + if (trustStore) { + trustKS = keystoresDir + "/trustKS.jks"; + createTrustStore(trustKS, trustPassword, certs); + } + + Configuration clientSSLConf = createSSLConfig( + SSLFactory.Mode.CLIENT, clientKS, clientPassword, clientPassword, trustKS); + Configuration serverSSLConf = createSSLConfig( + SSLFactory.Mode.SERVER, serverKS, serverPassword, serverPassword, trustKS); + + saveConfig(sslClientConfFile, clientSSLConf); + saveConfig(sslServerConfFile, serverSSLConf); + + conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL"); + conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName()); + conf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile.getName()); + conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert); + } + + /** + * Returns an SSL configuration file name. Under parallel test + * execution, this file name is parameterized by a unique ID to ensure that + * concurrent tests don't collide on an SSL configuration file. + * + * @param base the base of the file name + * @return SSL configuration file name for base + */ + private static String getSSLConfigFileName(String base) { + String testUniqueForkId = System.getProperty("test.unique.fork.id"); + String fileSuffix = testUniqueForkId != null ? "-" + testUniqueForkId : ""; + return base + fileSuffix + ".xml"; + } + + /** + * Creates SSL configuration. + * + * @param mode SSLFactory.Mode mode to configure + * @param keystore String keystore file + * @param password String store password, or null to avoid setting store + * password + * @param keyPassword String key password, or null to avoid setting key + * password + * @param trustKS String truststore file + * @return Configuration for SSL + */ + private static Configuration createSSLConfig(SSLFactory.Mode mode, + String keystore, String password, String keyPassword, String trustKS) { + String trustPassword = "trustP"; + + Configuration sslConf = new Configuration(false); + if (keystore != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), keystore); + } + if (password != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), password); + } + if (keyPassword != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY), + keyPassword); + } + if (trustKS != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS); + } + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY), + trustPassword); + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000"); + + return sslConf; + } + + private static KeyPair generateKeyPair(String algorithm) + throws NoSuchAlgorithmException { + KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm); + keyGen.initialize(1024); + return keyGen.genKeyPair(); + } + + /** + * Create a self-signed X.509 Certificate. + * + * @param dn the X.509 Distinguished Name, eg "CN=Test, L=London, C=GB" + * @param pair the KeyPair + * @param days how many days from now the Certificate is valid for + * @param algorithm the signing algorithm, eg "SHA1withRSA" + * @return the self-signed certificate + */ + private static X509Certificate generateCertificate(String dn, KeyPair pair, int days, String algorithm) + throws CertificateEncodingException, + InvalidKeyException, + IllegalStateException, + NoSuchProviderException, NoSuchAlgorithmException, SignatureException{ + Date from = new Date(); + Date to = new Date(from.getTime() + days * 86400000l); + BigInteger sn = new BigInteger(64, new SecureRandom()); + KeyPair keyPair = pair; + X509V1CertificateGenerator certGen = new X509V1CertificateGenerator(); + X500Principal dnName = new X500Principal(dn); + + certGen.setSerialNumber(sn); + certGen.setIssuerDN(dnName); + certGen.setNotBefore(from); + certGen.setNotAfter(to); + certGen.setSubjectDN(dnName); + certGen.setPublicKey(keyPair.getPublic()); + certGen.setSignatureAlgorithm(algorithm); + + X509Certificate cert = certGen.generate(pair.getPrivate()); + return cert; + } + + private static void createKeyStore(String filename, + String password, String alias, + Key privateKey, Certificate cert) + throws GeneralSecurityException, IOException { + KeyStore ks = KeyStore.getInstance("JKS"); + ks.load(null, null); // initialize + ks.setKeyEntry(alias, privateKey, password.toCharArray(), + new Certificate[]{cert}); + saveKeyStore(ks, filename, password); + } + + private static void createTrustStore( + String filename, String password, Map certs) + throws GeneralSecurityException, IOException { + KeyStore ks = KeyStore.getInstance("JKS"); + ks.load(null, null); // initialize + for (Map.Entry cert : certs.entrySet()) { + ks.setCertificateEntry(cert.getKey(), cert.getValue()); + } + saveKeyStore(ks, filename, password); + } + + private static void saveKeyStore(KeyStore ks, String filename, + String password) + throws GeneralSecurityException, IOException { + FileOutputStream out = new FileOutputStream(filename); + try { + ks.store(out, password.toCharArray()); + } finally { + out.close(); + } + } + + /** + * Saves configuration to a file. + * + * @param file File to save + * @param conf Configuration contents to write to file + * @throws IOException if there is an I/O error saving the file + */ + private static void saveConfig(File file, Configuration conf) + throws IOException { + Writer writer = new OutputStreamWriter(new FileOutputStream(file), "UTF-8"); + try { + conf.writeXml(writer); + } finally { + writer.close(); + } + } } diff --git a/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java deleted file mode 100644 index 35b9b7de..00000000 --- a/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java +++ /dev/null @@ -1,269 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.test.utils; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory; -import org.apache.hadoop.security.ssl.SSLFactory; -import org.bouncycastle.x509.X509V1CertificateGenerator; - -import javax.security.auth.x500.X500Principal; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.math.BigInteger; -import java.security.GeneralSecurityException; -import java.security.InvalidKeyException; -import java.security.Key; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.KeyStore; -import java.security.NoSuchAlgorithmException; -import java.security.NoSuchProviderException; -import java.security.SecureRandom; -import java.security.SignatureException; -import java.security.cert.Certificate; -import java.security.cert.CertificateEncodingException; -import java.security.cert.X509Certificate; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -public class SecurityUtils { - - public static final String CLIENT_KEYSTORE = "clientKS.jks"; - public static final String SERVER_KEYSTORE = "serverKS.jks"; - public static final String TRUSTSTORE = "trustKS.jks"; - - public static final String CLIENT_KEY_PASSWORD = "client_key"; - public static final String CLIENT_KEY_STORE_PASSWORD = "client_keystore"; - public static final String SERVER_KEY_PASSWORD = "server_key"; - public static final String SERVER_KEY_STORE_PASSWORD = "server_keystore"; - - /** - * Performs complete setup of SSL configuration. This includes keys, certs, - * keystores, truststores, the server SSL configuration file, - * the client SSL configuration file. - * - * @param keystoresDir String directory to save keystores - * @param sslConfDir String directory to save SSL configuration files - * @param conf Configuration - * @param useClientCert boolean true to make the client present a cert in the - * SSL handshake - * @param trustStore boolean true to create truststore, false not to create it - */ - public static X509Certificate setupSSLConfig(String keystoresDir, String sslConfDir, - Configuration conf, boolean useClientCert, boolean trustStore) - throws Exception { - String clientKeyStorePath = keystoresDir + "/" + CLIENT_KEYSTORE; - String serverKeyStorePath = keystoresDir + "/" + SERVER_KEYSTORE; - String serverPassword = "serverP"; - String trustKS = null; - String trustPassword = "trustP"; - - File sslClientConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-client")); - File sslServerConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-server")); - - Map certs = new HashMap(); - - String hostname = SqoopUtils.getLocalHostName(); - - if (useClientCert) { - KeyPair cKP = generateKeyPair("RSA"); - X509Certificate cCert = generateCertificate("CN=" + hostname + ", O=client", cKP, 30, "SHA1withRSA"); - createKeyStore(clientKeyStorePath, CLIENT_KEY_PASSWORD, CLIENT_KEY_STORE_PASSWORD, "client", cKP.getPrivate(), cCert); - certs.put("client", cCert); - } - - KeyPair sKP = generateKeyPair("RSA"); - X509Certificate sCert = generateCertificate("CN=" + hostname + ", O=server", sKP, 30, "SHA1withRSA"); - createKeyStore(serverKeyStorePath, SERVER_KEY_PASSWORD, SERVER_KEY_STORE_PASSWORD, "server", sKP.getPrivate(), sCert); - certs.put("server", sCert); - - if (trustStore) { - trustKS = keystoresDir + TRUSTSTORE; - createTrustStore(trustKS, trustPassword, certs); - } - - Configuration clientSSLConf = createSSLConfig( - SSLFactory.Mode.CLIENT, clientKeyStorePath, CLIENT_KEY_STORE_PASSWORD, CLIENT_KEY_PASSWORD, trustKS); - Configuration serverSSLConf = createSSLConfig( - SSLFactory.Mode.SERVER, serverKeyStorePath, SERVER_KEY_STORE_PASSWORD, SERVER_KEY_PASSWORD, trustKS); - - saveConfig(sslClientConfFile, clientSSLConf); - saveConfig(sslServerConfFile, serverSSLConf); - - conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL"); - conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName()); - conf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile.getName()); - conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert); - - return sCert; - } - - /** - * Returns an SSL configuration file name. Under parallel test - * execution, this file name is parameterized by a unique ID to ensure that - * concurrent tests don't collide on an SSL configuration file. - * - * @param base the base of the file name - * @return SSL configuration file name for base - */ - public static String getSSLConfigFileName(String base) { - String testUniqueForkId = System.getProperty("test.unique.fork.id"); - String fileSuffix = testUniqueForkId != null ? "-" + testUniqueForkId : ""; - return base + fileSuffix + ".xml"; - } - - /** - * Creates SSL configuration. - * - * @param mode SSLFactory.Mode mode to configure - * @param keystore String keystore file - * @param keyStorePassword String store password, or null to avoid setting store - * password - * @param keyPassword String key password, or null to avoid setting key - * password - * @param trustKS String truststore file - * @return Configuration for SSL - */ - public static Configuration createSSLConfig(SSLFactory.Mode mode, - String keystore, String keyStorePassword, - String keyPassword, String trustKS) { - String trustPassword = "trustP"; - - Configuration sslConf = new Configuration(false); - if (keystore != null) { - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), keystore); - } - - if (keyStorePassword != null) { - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), keyStorePassword); - } - if (keyPassword != null) { - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY), - keyPassword); - } - if (trustKS != null) { - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS); - } - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY), - trustPassword); - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000"); - - return sslConf; - } - - public static KeyPair generateKeyPair(String algorithm) - throws NoSuchAlgorithmException { - KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm); - keyGen.initialize(1024); - return keyGen.genKeyPair(); - } - - /** - * Create a self-signed X.509 Certificate. - * - * @param dn the X.509 Distinguished Name, eg "CN=Test, L=London, C=GB" - * @param pair the KeyPair - * @param days how many days from now the Certificate is valid for - * @param algorithm the signing algorithm, eg "SHA1withRSA" - * @return the self-signed certificate - */ - public static X509Certificate generateCertificate(String dn, KeyPair pair, int days, String algorithm) - throws CertificateEncodingException, - InvalidKeyException, - IllegalStateException, - NoSuchProviderException, NoSuchAlgorithmException, SignatureException { - Date from = new Date(); - Date to = new Date(from.getTime() + days * 86400000l); - BigInteger sn = new BigInteger(64, new SecureRandom()); - KeyPair keyPair = pair; - X509V1CertificateGenerator certGen = new X509V1CertificateGenerator(); - X500Principal dnName = new X500Principal(dn); - - certGen.setSerialNumber(sn); - certGen.setIssuerDN(dnName); - certGen.setNotBefore(from); - certGen.setNotAfter(to); - certGen.setSubjectDN(dnName); - certGen.setPublicKey(keyPair.getPublic()); - certGen.setSignatureAlgorithm(algorithm); - - X509Certificate cert = certGen.generate(pair.getPrivate()); - return cert; - } - - public static void createKeyStore(String filename, - String keyPassword, String keyStorePassword, - String alias, Key privateKey, Certificate cert) - throws GeneralSecurityException, IOException { - KeyStore ks = KeyStore.getInstance("JKS"); - ks.load(null, null); // initialize - ks.setKeyEntry(alias, privateKey, keyPassword.toCharArray(), - new Certificate[]{cert}); - saveKeyStore(ks, filename, keyStorePassword); - } - - public static void createTrustStore( - String filename, String password, Map certs) - throws GeneralSecurityException, IOException { - KeyStore ks = KeyStore.getInstance("JKS"); - ks.load(null, null); // initialize - for (Map.Entry cert : certs.entrySet()) { - ks.setCertificateEntry(cert.getKey(), cert.getValue()); - } - saveKeyStore(ks, filename, password); - } - - public static void saveKeyStore(KeyStore ks, String filename, - String password) - throws GeneralSecurityException, IOException { - FileOutputStream out = new FileOutputStream(filename); - try { - ks.store(out, password.toCharArray()); - } finally { - out.close(); - } - } - - /** - * Saves configuration to a file. - * - * @param file File to save - * @param conf Configuration contents to write to file - * @throws IOException if there is an I/O error saving the file - */ - public static void saveConfig(File file, Configuration conf) - throws IOException { - Writer writer = new OutputStreamWriter(new FileOutputStream(file), "UTF-8"); - try { - conf.writeXml(writer); - } finally { - writer.close(); - } - } -} diff --git a/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java index 3e0566f4..6614b199 100644 --- a/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java +++ b/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java @@ -18,7 +18,6 @@ package org.apache.sqoop.test.utils; import java.net.InetAddress; -import java.net.URL; import java.net.UnknownHostException; import java.util.Locale; import java.util.Random; @@ -45,7 +44,7 @@ public static void fillObjectName(MAccountableEntity object) { object.setName(prefix + rand.nextLong()); } - // Retrieve the FQDN of the current host + //Retrieve the FQDN of the current host public static String getLocalHostName() { String fqdn; try { @@ -55,26 +54,4 @@ public static String getLocalHostName() { } return fqdn; } - - // Retrieve the IP address of the current host - public static String getLocalIpAddress() { - String address; - try { - address = InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException e1) { - address = "127.0.0.1"; - } - return address; - } - - - @SuppressWarnings("rawtypes") - public static String getClasspathDir(Class klass) throws Exception { - String file = klass.getName(); - file = file.replace('.', '/') + ".class"; - URL url = Thread.currentThread().getContextClassLoader().getResource(file); - String baseDir = url.toURI().getPath(); - baseDir = baseDir.substring(0, baseDir.length() - file.length() - 1); - return baseDir; - } } diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java similarity index 98% rename from test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java rename to test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java index 438e1823..7f0575b4 100644 --- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.sqoop.integration.serverproperties; +package org.apache.sqoop.integration.connectorloading; import org.apache.hadoop.conf.Configuration; import org.apache.sqoop.common.SqoopException; diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java similarity index 99% rename from test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java rename to test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java index 1a332302..87f0eb19 100644 --- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.sqoop.integration.serverproperties; +package org.apache.sqoop.integration.connectorloading; import org.apache.hadoop.conf.Configuration; import org.apache.sqoop.core.ConfigurationConstants; diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java similarity index 99% rename from test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java rename to test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java index 18292575..a82a4dae 100644 --- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.sqoop.integration.serverproperties; +package org.apache.sqoop.integration.connectorloading; import java.util.Arrays; import java.util.HashMap; diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java b/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java deleted file mode 100644 index 17503f3f..00000000 --- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.integration.serverproperties; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; -import org.apache.sqoop.security.SecurityConstants; -import org.apache.sqoop.test.infrastructure.Infrastructure; -import org.apache.sqoop.test.infrastructure.SqoopTestCase; -import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider; -import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; -import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; -import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; -import org.apache.sqoop.test.minicluster.SqoopMiniCluster; -import org.apache.sqoop.test.utils.HdfsUtils; -import org.apache.sqoop.test.utils.SecurityUtils; -import org.apache.sqoop.test.utils.SqoopUtils; -import org.eclipse.jetty.util.ssl.SslContextFactory; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; -import javax.ws.rs.HttpMethod; -import javax.ws.rs.core.MediaType; -import java.io.File; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; -import java.security.cert.Certificate; -import java.security.cert.X509Certificate; -import java.util.Map; - -import static org.testng.Assert.assertEquals; - -@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class, DatabaseInfrastructureProvider.class}) -@Test(groups = {"no-real-cluster"}) -public class SslTest extends SqoopTestCase { - - private SqoopMiniCluster sqoopMiniCluster; - private SSLContext defaultSslContext; - private HostnameVerifier defaultHostNameVerifier; - - public static class SslSqoopMiniCluster extends JettySqoopMiniCluster { - - private String keyStoreFilePath; - private String keyStorePassword; - private String keyManagerPassword; - - public SslSqoopMiniCluster(String temporaryPath, Configuration configuration, String keyStoreFilePath, String keyStorePassword, String keyManagerPassword) throws Exception { - super(temporaryPath, configuration); - this.keyStoreFilePath = keyStoreFilePath; - this.keyStorePassword = keyStorePassword; - this.keyManagerPassword = keyManagerPassword; - } - - @Override - protected Map getSecurityConfiguration() { - Map properties = super.getSecurityConfiguration(); - - properties.put(SecurityConstants.TLS_ENABLED, String.valueOf(true)); - properties.put(SecurityConstants.TLS_PROTOCOL, "TLSv1.2"); - properties.put(SecurityConstants.KEYSTORE_LOCATION, keyStoreFilePath); - properties.put(SecurityConstants.KEYSTORE_PASSWORD, keyStorePassword); - properties.put(SecurityConstants.KEYMANAGER_PASSWORD, keyManagerPassword); - - return properties; - } - } - - @BeforeMethod - public void backupSslContext() throws Exception { - defaultSslContext = SSLContext.getDefault(); - defaultHostNameVerifier = HttpsURLConnection.getDefaultHostnameVerifier(); - } - - @AfterMethod - public void restoreSslContext() { - SSLContext.setDefault(defaultSslContext); - HttpsURLConnection.setDefaultHostnameVerifier(defaultHostNameVerifier); - } - - @AfterMethod - public void stopCluster() throws Exception { - sqoopMiniCluster.stop(); - } - - @Test - public void testSslInUse() throws Exception { - String sslKeystoresDir = getTemporaryPath() + "ssl-keystore/"; - String sslConfDir = SqoopUtils.getClasspathDir(SslTest.class); - FileUtils.deleteDirectory(new File(sslKeystoresDir)); - FileUtils.forceMkdir(new File(sslKeystoresDir)); - X509Certificate serverCertificate = SecurityUtils.setupSSLConfig( - sslKeystoresDir, sslConfDir, new Configuration(), false, true); - - sqoopMiniCluster = - new SslSqoopMiniCluster(HdfsUtils.joinPathFragments(getTemporaryPath(), getTestName()), getHadoopConf(), sslKeystoresDir + SecurityUtils.SERVER_KEYSTORE, SecurityUtils.SERVER_KEY_STORE_PASSWORD, SecurityUtils.SERVER_KEY_PASSWORD); - - KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class); - if (kdcProvider != null) { - sqoopMiniCluster.setKdc(kdcProvider.getInstance()); - } - - sqoopMiniCluster.start(); - - // Bypass hostname verification - HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() { - public boolean verify(String hostname, SSLSession session) { - try { - if (hostname.equals((new URL(sqoopMiniCluster.getServerUrl())).getHost())) { - return true; - } - } catch (MalformedURLException e) { - return false; - } - return false; - } - }); - - SslContextFactory sslContextFactory = new SslContextFactory(); - sslContextFactory.setKeyStorePath(sslKeystoresDir + SecurityUtils.TRUSTSTORE); - - sslContextFactory.start(); - - SSLContext.setDefault(sslContextFactory.getSslContext()); - - initSqoopClient(sqoopMiniCluster.getServerUrl()); - - // Make a request and check the cert - URL url = new URL(sqoopMiniCluster.getServerUrl() + "version?" + - PseudoAuthenticator.USER_NAME + "=" + System.getProperty("user.name")); - HttpURLConnection conn = new DelegationTokenAuthenticatedURL().openConnection(url, getAuthToken()); - conn.setRequestMethod(HttpMethod.GET); - conn.setRequestProperty("Accept", MediaType.APPLICATION_JSON); - - assertEquals(conn.getResponseCode(), 200); - - HttpsURLConnection secured = (HttpsURLConnection) conn; - Certificate actualCertificate = secured.getServerCertificates()[0]; - assertEquals(actualCertificate, serverCertificate); - } - -} diff --git a/test/src/test/resources/server-properties-tests-suite.xml b/test/src/test/resources/connector-loading-tests-suite.xml similarity index 86% rename from test/src/test/resources/server-properties-tests-suite.xml rename to test/src/test/resources/connector-loading-tests-suite.xml index 2743996a..c03fb4f0 100644 --- a/test/src/test/resources/server-properties-tests-suite.xml +++ b/test/src/test/resources/connector-loading-tests-suite.xml @@ -18,16 +18,16 @@ limitations under the License. - + - + - +