From 99223379bf4eb1b8be4421a1cb82f2af12f15c58 Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Tue, 8 Mar 2016 12:39:28 -0800 Subject: [PATCH] SQOOP-2844: Sqoop2: TrustStore support for shell (Abraham Fine via Jarek Jarcec Cecho) --- .../org/apache/sqoop/utils/ProcessUtils.java | 40 ++++++++ .../sqoop/handler/DriverRequestHandler.java | 5 + .../apache/sqoop/server/SqoopJettyServer.java | 25 +---- .../org/apache/sqoop/shell/SetCommand.java | 3 +- .../apache/sqoop/shell/SetServerFunction.java | 12 ++- .../sqoop/shell/SetTruststoreFunction.java | 98 +++++++++++++++++++ .../apache/sqoop/shell/ShellEnvironment.java | 9 +- .../apache/sqoop/shell/core/Constants.java | 19 ++++ .../main/resources/shell-resource.properties | 7 +- .../server/rest/DriverRestTest.java | 53 ---------- 10 files changed, 192 insertions(+), 79 deletions(-) create mode 100644 common/src/main/java/org/apache/sqoop/utils/ProcessUtils.java create mode 100644 shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java delete mode 100644 test/src/test/java/org/apache/sqoop/integration/server/rest/DriverRestTest.java diff --git a/common/src/main/java/org/apache/sqoop/utils/ProcessUtils.java b/common/src/main/java/org/apache/sqoop/utils/ProcessUtils.java new file mode 100644 index 00000000..0a32d686 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/utils/ProcessUtils.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.utils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; + +public class ProcessUtils { + public static String readOutputFromGenerator(String generatorCommand) throws IOException { + ProcessBuilder processBuilder = new ProcessBuilder("/bin/sh", "-c", generatorCommand); + Process process = processBuilder.start(); + String output; + try ( + InputStreamReader inputStreamReader = new InputStreamReader(process.getInputStream(), Charset.forName("UTF-8")); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader); + ) { + output = bufferedReader.readLine(); + } catch(IOException exception) { + throw exception; + } + return output; + } +} diff --git a/server/src/main/java/org/apache/sqoop/handler/DriverRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/DriverRequestHandler.java index 7b82ce54..95a32916 100644 --- a/server/src/main/java/org/apache/sqoop/handler/DriverRequestHandler.java +++ b/server/src/main/java/org/apache/sqoop/handler/DriverRequestHandler.java @@ -40,6 +40,11 @@ public DriverRequestHandler() { @Override public JsonBean handleEvent(RequestContext ctx) { + // driver only support GET requests + if (ctx.getMethod() != Method.GET) { + throw new SqoopException(ServerError.SERVER_0002, "Unsupported HTTP method for driver:" + + ctx.getMethod()); + } AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(), "get", "driver", ""); diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java index 4696a876..00aa2148 100644 --- a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java +++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java @@ -34,6 +34,7 @@ import org.apache.sqoop.server.v1.JobServlet; import org.apache.sqoop.server.v1.LinkServlet; import org.apache.sqoop.server.v1.SubmissionsServlet; +import org.apache.sqoop.utils.ProcessUtils; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.SecureRequestCustomizer; @@ -45,10 +46,7 @@ import org.eclipse.jetty.util.thread.ExecutorThreadPool; import javax.servlet.DispatcherType; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.Charset; import java.util.EnumSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; @@ -106,7 +104,7 @@ public SqoopJettyServer() { sslContextFactory.setKeyStorePassword(keyStorePassword); } else if (StringUtils.isNotBlank(keyStorePasswordGenerator)) { try { - String passwordFromGenerator = readPasswordFromGenerator(keyStorePasswordGenerator); + String passwordFromGenerator = ProcessUtils.readOutputFromGenerator(keyStorePasswordGenerator); sslContextFactory.setKeyStorePassword(passwordFromGenerator); } catch (IOException exception) { throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator: " + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR, exception); @@ -123,7 +121,7 @@ public SqoopJettyServer() { } } else if (StringUtils.isNotBlank(keyManagerPasswordGenerator)) { try { - String passwordFromGenerator = readPasswordFromGenerator(keyManagerPasswordGenerator); + String passwordFromGenerator = ProcessUtils.readOutputFromGenerator(keyManagerPasswordGenerator); sslContextFactory.setKeyManagerPassword(passwordFromGenerator); } catch (IOException exception) { throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator: " + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR, exception); @@ -146,21 +144,6 @@ public SqoopJettyServer() { webServer.setHandler(createServletContextHandler()); } - private String readPasswordFromGenerator(String generatorCommand) throws IOException { - ProcessBuilder processBuilder = new ProcessBuilder("/bin/sh", "-c", generatorCommand); - Process process = processBuilder.start(); - String output; - try ( - InputStreamReader inputStreamReader = new InputStreamReader(process.getInputStream(), Charset.forName("UTF-8")); - BufferedReader bufferedReader = new BufferedReader(inputStreamReader); - ) { - output = bufferedReader.readLine(); - } catch(IOException exception) { - throw exception; - } - return output; - } - public synchronized void startServer() { try { webServer.start(); @@ -212,7 +195,7 @@ private static ServletContextHandler createServletContextHandler() { return context; } - public static void main(String[] args){ + public static void main(String[] args) { SqoopJettyServer sqoopJettyServer = new SqoopJettyServer(); sqoopJettyServer.startServer(); sqoopJettyServer.joinServerThread(); diff --git a/shell/src/main/java/org/apache/sqoop/shell/SetCommand.java b/shell/src/main/java/org/apache/sqoop/shell/SetCommand.java index 3feaac30..20336716 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/SetCommand.java +++ b/shell/src/main/java/org/apache/sqoop/shell/SetCommand.java @@ -29,7 +29,8 @@ public SetCommand(Groovysh shell) { Constants.CMD_SET_SC, ImmutableMap.of( Constants.FN_SERVER, SetServerFunction.class, - Constants.FN_OPTION, SetOptionFunction.class + Constants.FN_OPTION, SetOptionFunction.class, + Constants.FN_TRUSTSTORE, SetTruststoreFunction.class ) ); } diff --git a/shell/src/main/java/org/apache/sqoop/shell/SetServerFunction.java b/shell/src/main/java/org/apache/sqoop/shell/SetServerFunction.java index e430f9d1..a0fc1667 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/SetServerFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/SetServerFunction.java @@ -46,6 +46,10 @@ public SetServerFunction() { .withDescription(resourceString(Constants.RES_URL_DESCRIPTION)) .withLongOpt(Constants.OPT_URL) .create(Constants.OPT_URL_CHAR)); + this.addOption(OptionBuilder.withArgName(Constants.OPT_TLS) + .withDescription(resourceString(Constants.RES_TLS_DESCRIPTION)) + .withLongOpt(Constants.OPT_TLS) + .create()); } @Override @@ -62,15 +66,16 @@ public Object executeFunction(CommandLine line, boolean isInteractive) { if (line.hasOption(Constants.OPT_URL)) { setServerUrl(line.getOptionValue(Constants.OPT_URL)); - // ignore --host, --port and --webapp option + // ignore --host, --tls, --port and --webapp option if (line.hasOption(Constants.OPT_HOST) + || line.hasOption(Constants.OPT_TLS) || line.hasOption(Constants.OPT_PORT) || line.hasOption(Constants.OPT_WEBAPP)) { printlnResource(Constants.RES_SET_SERVER_IGNORED); } } else { if (line.hasOption(Constants.OPT_HOST)) { - setServerHost(line.getOptionValue(Constants.OPT_HOST)); + setServerHost(line.getOptionValue(Constants.OPT_HOST)); } if (line.hasOption(Constants.OPT_PORT)) { setServerPort(line.getOptionValue(Constants.OPT_PORT)); @@ -78,6 +83,9 @@ public Object executeFunction(CommandLine line, boolean isInteractive) { if (line.hasOption(Constants.OPT_WEBAPP)) { setServerWebapp(line.getOptionValue(Constants.OPT_WEBAPP)); } + if (line.hasOption(Constants.OPT_TLS)) { + setServerTls(); + } } printlnResource(Constants.RES_SET_SERVER_SUCCESSFUL); diff --git a/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java b/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java new file mode 100644 index 00000000..0b603a1d --- /dev/null +++ b/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.shell; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.OptionBuilder; +import org.apache.sqoop.shell.core.Constants; +import org.apache.sqoop.utils.ProcessUtils; +import org.apache.sqoop.validation.Status; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManagerFactory; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; + +import static org.apache.sqoop.shell.ShellEnvironment.printlnResource; +import static org.apache.sqoop.shell.ShellEnvironment.resourceString; + +@SuppressWarnings("serial") +public class SetTruststoreFunction extends SqoopFunction { + private static final long serialVersionUID = 1L; + + @SuppressWarnings("static-access") + public SetTruststoreFunction() { + this.addOption(OptionBuilder.hasArg().withArgName(Constants.OPT_TRUSTSTORE) + .withDescription(resourceString(Constants.RES_TRUSTSTORE_DESCRIPTION)) + .withLongOpt(Constants.OPT_TRUSTSTORE) + .create()); + this.addOption(OptionBuilder.hasArg().withArgName(Constants.OPT_TRUSTSTORE_PASSWORD) + .withDescription(resourceString(Constants.RES_TRUSTSTORE_PASSWORD_DESCRIPTION)) + .withLongOpt(Constants.OPT_TRUSTSTORE_PASSWORD) + .create()); + this.addOption(OptionBuilder.hasArg().withArgName(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR) + .withDescription(resourceString(Constants.RES_TRUSTSTORE_PASSWORD_GENERATOR_DESCRIPTION)) + .withLongOpt(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR) + .create()); + } + + @Override + public Object executeFunction(CommandLine line, boolean isInteractive) throws IOException { + try { + if (line.hasOption(Constants.OPT_TRUSTSTORE)) { + String truststoreLocation = line.getOptionValue(Constants.OPT_TRUSTSTORE); + + char[] truststorePassword = null; + if (line.hasOption(Constants.OPT_TRUSTSTORE_PASSWORD)) { + truststorePassword = line.getOptionValue(Constants.OPT_TRUSTSTORE_PASSWORD).toCharArray(); + } else if (line.hasOption(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR)) { + String generator = line.getOptionValue(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR); + truststorePassword = ProcessUtils.readOutputFromGenerator(generator).toCharArray(); + } + + KeyStore keyStore = KeyStore.getInstance("JKS"); + + File truststore = new File(truststoreLocation); + try (FileInputStream trustStoreFileInputStream = new FileInputStream(truststore)) { + keyStore.load(trustStoreFileInputStream, truststorePassword); + } + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, trustManagerFactory.getTrustManagers(), null); + SSLSocketFactory sslSocketFactory = context.getSocketFactory(); + HttpsURLConnection.setDefaultSSLSocketFactory(sslSocketFactory); + } + } catch (IOException|CertificateException|NoSuchAlgorithmException|KeyStoreException|KeyManagementException exception) { + printlnResource(Constants.RES_SET_TRUSTSTORE_FAILED); + throw new IOException("failed to set truststore", exception); + } + + printlnResource(Constants.RES_SET_TRUSTSTORE_SUCCESSFUL); + return Status.OK; + } +} diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java b/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java index 80ac935e..ce31dd0c 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java @@ -50,10 +50,12 @@ private ShellEnvironment() { private static String DEFAULT_SERVER_HOST = getEnv(Constants.ENV_HOST, "localhost"); private static String DEFAULT_SERVER_PORT = getEnv(Constants.ENV_PORT, "12000"); private static String DEFAULT_SERVER_WEBAPP = getEnv(Constants.ENV_WEBAPP, "sqoop"); + private static String DEFAULT_PROTOCOL = getEnv(Constants.ENV_PROTOCOL, "http"); private static String serverHost = DEFAULT_SERVER_HOST; private static String serverPort = DEFAULT_SERVER_PORT; private static String serverWebapp = DEFAULT_SERVER_WEBAPP; + private static String protocol = DEFAULT_PROTOCOL; private static boolean verbose = false; private static boolean interactive = false; @@ -120,6 +122,11 @@ public static void setServerWebapp(String webapp) { client.setServerUrl(getServerUrl()); } + public static void setServerTls() { + protocol = "https"; + client.setServerUrl(getServerUrl()); + } + public static String getServerWebapp() { return serverWebapp; } @@ -157,7 +164,7 @@ public static void setServerUrl(String ustr){ } public static String getServerUrl() { - return "http://" + serverHost + ":" + serverPort + "/" + serverWebapp + "/"; + return protocol + "://" + serverHost + ":" + serverPort + "/" + serverWebapp + "/"; } public static ResourceBundle getResourceBundle() { diff --git a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java index 8af53f21..a5f5d4d0 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java +++ b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java @@ -31,6 +31,7 @@ public class Constants { public static final String ENV_HOST = "SQOOP2_HOST"; public static final String ENV_PORT = "SQOOP2_PORT"; public static final String ENV_WEBAPP = "SQOOP2_WEBAPP"; + public static final String ENV_PROTOCOL = "SQOOP2_PROTOCOL"; // Options @@ -42,6 +43,7 @@ public class Constants { public static final String OPT_VALUE = "value"; public static final String OPT_VERBOSE = "verbose"; public static final String OPT_HOST = "host"; + public static final String OPT_TLS = "tls"; public static final String OPT_PORT = "port"; public static final String OPT_WEBAPP = "webapp"; public static final String OPT_URL = "url"; @@ -59,6 +61,9 @@ public class Constants { public static final String OPT_PRINCIPAL_TYPE = "principal-type"; public static final String OPT_WITH_GRANT = "with-grant"; public static final String OPT_WITH_JOB = "job"; + public static final String OPT_TRUSTSTORE = "truststore"; + public static final String OPT_TRUSTSTORE_PASSWORD = "truststore-password"; + public static final String OPT_TRUSTSTORE_PASSWORD_GENERATOR = "truststore-password-generator"; public static final char OPT_FROM_CHAR = 'f'; public static final char OPT_TO_CHAR = 't'; @@ -80,6 +85,7 @@ public class Constants { public static final char OPT_ACTION_CHAR = 'a'; public static final char OPT_WITH_GRANT_CHAR = 'g'; public static final char OPT_WITH_JOB_CHAR = 'j'; + public static final char OPT_TRUSTSTORE_CHAR2 = 'j'; // Resource keys for various commands, command options, // functions and descriptions @@ -130,6 +136,7 @@ public class Constants { public static final String FN_SUBMISSION = "submission"; public static final String FN_SERVER = "server"; public static final String FN_OPTION = "option"; + public static final String FN_TRUSTSTORE = "truststore"; public static final String FN_CONNECTOR = "connector"; public static final String FN_VERSION = "version"; public static final String FN_DRIVER_CONFIG = "driver"; @@ -248,12 +255,24 @@ public class Constants { "set.webapp_description"; public static final String RES_URL_DESCRIPTION = "set.url_description"; + public static final String RES_TLS_DESCRIPTION = + "set.tls_description"; public static final String RES_SET_SERVER_USAGE = "set.server_usage"; public static final String RES_SET_SERVER_SUCCESSFUL = "set.server_successful"; public static final String RES_SET_SERVER_IGNORED = "set.server_ignored"; + public static final String RES_SET_TRUSTSTORE_SUCCESSFUL = + "set.truststore_successful"; + public static final String RES_SET_TRUSTSTORE_FAILED= + "set.truststore_failed"; + public static final String RES_TRUSTSTORE_DESCRIPTION = + "set.truststore_description"; + public static final String RES_TRUSTSTORE_PASSWORD_DESCRIPTION = + "set.truststore_password_description"; + public static final String RES_TRUSTSTORE_PASSWORD_GENERATOR_DESCRIPTION = + "set.truststore_password_generator_description"; public static final String RES_SHOW_PROMPT_DISPLAY_ALL_LINKS = "show.prompt_display_all_links"; diff --git a/shell/src/main/resources/shell-resource.properties b/shell/src/main/resources/shell-resource.properties index 630c31d7..2c94c584 100644 --- a/shell/src/main/resources/shell-resource.properties +++ b/shell/src/main/resources/shell-resource.properties @@ -121,10 +121,15 @@ set.host_description = Host name to invoke server resources set.port_description = Port number to invoke server resources set.webapp_description = Web app to invoke server resources set.url_description = Url to invoke server resources +set.tls_description = Flag that determines if https should be used instead of http set.server_usage = Usage: set server set.server_successful = Server is set successfully set.server_ignored = --host, --port or --webapp option is ignored, because --url option is given. - +set.truststore_successful = Truststore set successfully +set.truststore_failed = Failed to set the truststore +set.truststore_description = Path to the truststore +set.truststore_password_description = Password for the truststore +set.truststore_password_generator_description = Command that prints the password to the truststore # Show command show.description = Display various objects and configuration options diff --git a/test/src/test/java/org/apache/sqoop/integration/server/rest/DriverRestTest.java b/test/src/test/java/org/apache/sqoop/integration/server/rest/DriverRestTest.java deleted file mode 100644 index 438e074f..00000000 --- a/test/src/test/java/org/apache/sqoop/integration/server/rest/DriverRestTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.integration.server.rest; - -import org.apache.sqoop.test.utils.ParametrizedUtils; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Factory; - -import java.util.Iterator; - -public class DriverRestTest extends RestTest { - - public static TestDescription[] PROVIDER_DATA = new TestDescription[]{ - new TestDescription("Get driver", "v1/driver", "GET", null, new Validator() { - @Override - void validate() throws Exception { - assertResponseCode(200); - assertContains("job-config"); - assertContains("all-config-resources"); - }}), - new TestDescription("Invalid post request", "v1/driver", "POST", "Random data", new Validator() { - @Override - void validate() throws Exception { - assertResponseCode(405); - assertServerException("Unsupported HTTP method", "SERVER_0002"); - }}), - }; - - @DataProvider(name="driver-rest-test") - public static Iterator data() { - return ParametrizedUtils.toArrayOfArrays(PROVIDER_DATA).iterator(); - } - - @Factory(dataProvider = "driver-rest-test") - public DriverRestTest(TestDescription desc) { - super(desc); - } -}