github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-09 00:29:40 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-2844: Sqoop2: TrustStore support for shell

Browse Source 
(Abraham Fine via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2016-03-08 12:39:28 -08:00
parent 004b298e38
commit 99223379bf
10 changed files with 192 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
common/src/main/java/org/apache/sqoop/utils/ProcessUtils.java Normal file
View File

@ -0,0 +1,40 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.utils;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
public class ProcessUtils {
public static String readOutputFromGenerator(String generatorCommand) throws IOException {
ProcessBuilder processBuilder = new ProcessBuilder("/bin/sh", "-c", generatorCommand);
Process process = processBuilder.start();
String output;
try (
InputStreamReader inputStreamReader = new InputStreamReader(process.getInputStream(), Charset.forName("UTF-8"));
BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
) {
output = bufferedReader.readLine();
} catch(IOException exception) {
throw exception;
}
return output;
}
}

5
server/src/main/java/org/apache/sqoop/handler/DriverRequestHandler.java
View File

@ -40,6 +40,11 @@ public DriverRequestHandler() {
@Override
public JsonBean handleEvent(RequestContext ctx) {
// driver only support GET requests
if (ctx.getMethod() != Method.GET) {
throw new SqoopException(ServerError.SERVER_0002, "Unsupported HTTP method for driver:"
+ ctx.getMethod());
}
AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(),
ctx.getRequest().getRemoteAddr(), "get", "driver", "");

25
server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
View File

@ -34,6 +34,7 @@
import org.apache.sqoop.server.v1.JobServlet;
import org.apache.sqoop.server.v1.LinkServlet;
import org.apache.sqoop.server.v1.SubmissionsServlet;
import org.apache.sqoop.utils.ProcessUtils;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.SecureRequestCustomizer;
@ -45,10 +46,7 @@
import org.eclipse.jetty.util.thread.ExecutorThreadPool;
import javax.servlet.DispatcherType;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.EnumSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
@ -106,7 +104,7 @@ public SqoopJettyServer() {
sslContextFactory.setKeyStorePassword(keyStorePassword);
} else if (StringUtils.isNotBlank(keyStorePasswordGenerator)) {
try {
String passwordFromGenerator = readPasswordFromGenerator(keyStorePasswordGenerator);
String passwordFromGenerator = ProcessUtils.readOutputFromGenerator(keyStorePasswordGenerator);
sslContextFactory.setKeyStorePassword(passwordFromGenerator);
} catch (IOException exception) {
throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator: " + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR, exception);
@ -123,7 +121,7 @@ public SqoopJettyServer() {
}
} else if (StringUtils.isNotBlank(keyManagerPasswordGenerator)) {
try {
String passwordFromGenerator = readPasswordFromGenerator(keyManagerPasswordGenerator);
String passwordFromGenerator = ProcessUtils.readOutputFromGenerator(keyManagerPasswordGenerator);
sslContextFactory.setKeyManagerPassword(passwordFromGenerator);
} catch (IOException exception) {
throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator: " + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR, exception);
@ -146,21 +144,6 @@ public SqoopJettyServer() {
webServer.setHandler(createServletContextHandler());
}
private String readPasswordFromGenerator(String generatorCommand) throws IOException {
ProcessBuilder processBuilder = new ProcessBuilder("/bin/sh", "-c", generatorCommand);
Process process = processBuilder.start();
String output;
try (
InputStreamReader inputStreamReader = new InputStreamReader(process.getInputStream(), Charset.forName("UTF-8"));
BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
) {
output = bufferedReader.readLine();
} catch(IOException exception) {
throw exception;
}
return output;
}
public synchronized void startServer() {
try {
webServer.start();
@ -212,7 +195,7 @@ private static ServletContextHandler createServletContextHandler() {
return context;
}
public static void main(String[] args){
public static void main(String[] args) {
SqoopJettyServer sqoopJettyServer = new SqoopJettyServer();
sqoopJettyServer.startServer();
sqoopJettyServer.joinServerThread();

3
shell/src/main/java/org/apache/sqoop/shell/SetCommand.java
View File

@ -29,7 +29,8 @@ public SetCommand(Groovysh shell) {
Constants.CMD_SET_SC,
ImmutableMap.of(
Constants.FN_SERVER, SetServerFunction.class,
Constants.FN_OPTION, SetOptionFunction.class
Constants.FN_OPTION, SetOptionFunction.class,
Constants.FN_TRUSTSTORE, SetTruststoreFunction.class
)
);
}

12
shell/src/main/java/org/apache/sqoop/shell/SetServerFunction.java
View File

@ -46,6 +46,10 @@ public SetServerFunction() {
.withDescription(resourceString(Constants.RES_URL_DESCRIPTION))
.withLongOpt(Constants.OPT_URL)
.create(Constants.OPT_URL_CHAR));
this.addOption(OptionBuilder.withArgName(Constants.OPT_TLS)
.withDescription(resourceString(Constants.RES_TLS_DESCRIPTION))
.withLongOpt(Constants.OPT_TLS)
.create());
}
@Override
@ -62,15 +66,16 @@ public Object executeFunction(CommandLine line, boolean isInteractive) {
if (line.hasOption(Constants.OPT_URL)) {
setServerUrl(line.getOptionValue(Constants.OPT_URL));
// ignore --host, --port and --webapp option
// ignore --host, --tls, --port and --webapp option
if (line.hasOption(Constants.OPT_HOST)
|| line.hasOption(Constants.OPT_TLS)
|| line.hasOption(Constants.OPT_PORT)
|| line.hasOption(Constants.OPT_WEBAPP)) {
printlnResource(Constants.RES_SET_SERVER_IGNORED);
}
} else {
if (line.hasOption(Constants.OPT_HOST)) {
setServerHost(line.getOptionValue(Constants.OPT_HOST));
setServerHost(line.getOptionValue(Constants.OPT_HOST));
}
if (line.hasOption(Constants.OPT_PORT)) {
setServerPort(line.getOptionValue(Constants.OPT_PORT));
@ -78,6 +83,9 @@ public Object executeFunction(CommandLine line, boolean isInteractive) {
if (line.hasOption(Constants.OPT_WEBAPP)) {
setServerWebapp(line.getOptionValue(Constants.OPT_WEBAPP));
}
if (line.hasOption(Constants.OPT_TLS)) {
setServerTls();
}
}
printlnResource(Constants.RES_SET_SERVER_SUCCESSFUL);

98
shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java Normal file
View File

@ -0,0 +1,98 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.shell;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.shell.core.Constants;
import org.apache.sqoop.utils.ProcessUtils;
import org.apache.sqoop.validation.Status;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManagerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import static org.apache.sqoop.shell.ShellEnvironment.printlnResource;
import static org.apache.sqoop.shell.ShellEnvironment.resourceString;
@SuppressWarnings("serial")
public class SetTruststoreFunction extends SqoopFunction {
private static final long serialVersionUID = 1L;
@SuppressWarnings("static-access")
public SetTruststoreFunction() {
this.addOption(OptionBuilder.hasArg().withArgName(Constants.OPT_TRUSTSTORE)
.withDescription(resourceString(Constants.RES_TRUSTSTORE_DESCRIPTION))
.withLongOpt(Constants.OPT_TRUSTSTORE)
.create());
this.addOption(OptionBuilder.hasArg().withArgName(Constants.OPT_TRUSTSTORE_PASSWORD)
.withDescription(resourceString(Constants.RES_TRUSTSTORE_PASSWORD_DESCRIPTION))
.withLongOpt(Constants.OPT_TRUSTSTORE_PASSWORD)
.create());
this.addOption(OptionBuilder.hasArg().withArgName(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR)
.withDescription(resourceString(Constants.RES_TRUSTSTORE_PASSWORD_GENERATOR_DESCRIPTION))
.withLongOpt(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR)
.create());
}
@Override
public Object executeFunction(CommandLine line, boolean isInteractive) throws IOException {
try {
if (line.hasOption(Constants.OPT_TRUSTSTORE)) {
String truststoreLocation = line.getOptionValue(Constants.OPT_TRUSTSTORE);
char[] truststorePassword = null;
if (line.hasOption(Constants.OPT_TRUSTSTORE_PASSWORD)) {
truststorePassword = line.getOptionValue(Constants.OPT_TRUSTSTORE_PASSWORD).toCharArray();
} else if (line.hasOption(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR)) {
String generator = line.getOptionValue(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR);
truststorePassword = ProcessUtils.readOutputFromGenerator(generator).toCharArray();
}
KeyStore keyStore = KeyStore.getInstance("JKS");
File truststore = new File(truststoreLocation);
try (FileInputStream trustStoreFileInputStream = new FileInputStream(truststore)) {
keyStore.load(trustStoreFileInputStream, truststorePassword);
}
TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
trustManagerFactory.init(keyStore);
SSLContext context = SSLContext.getInstance("TLS");
context.init(null, trustManagerFactory.getTrustManagers(), null);
SSLSocketFactory sslSocketFactory = context.getSocketFactory();
HttpsURLConnection.setDefaultSSLSocketFactory(sslSocketFactory);
}
} catch (IOException|CertificateException|NoSuchAlgorithmException|KeyStoreException|KeyManagementException exception) {
printlnResource(Constants.RES_SET_TRUSTSTORE_FAILED);
throw new IOException("failed to set truststore", exception);
}
printlnResource(Constants.RES_SET_TRUSTSTORE_SUCCESSFUL);
return Status.OK;
}
}

9
shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java
View File

@ -50,10 +50,12 @@ private ShellEnvironment() {
private static String DEFAULT_SERVER_HOST = getEnv(Constants.ENV_HOST, "localhost");
private static String DEFAULT_SERVER_PORT = getEnv(Constants.ENV_PORT, "12000");
private static String DEFAULT_SERVER_WEBAPP = getEnv(Constants.ENV_WEBAPP, "sqoop");
private static String DEFAULT_PROTOCOL = getEnv(Constants.ENV_PROTOCOL, "http");
private static String serverHost = DEFAULT_SERVER_HOST;
private static String serverPort = DEFAULT_SERVER_PORT;
private static String serverWebapp = DEFAULT_SERVER_WEBAPP;
private static String protocol = DEFAULT_PROTOCOL;
private static boolean verbose = false;
private static boolean interactive = false;
@ -120,6 +122,11 @@ public static void setServerWebapp(String webapp) {
client.setServerUrl(getServerUrl());
}
public static void setServerTls() {
protocol = "https";
client.setServerUrl(getServerUrl());
}
public static String getServerWebapp() {
return serverWebapp;
}
@ -157,7 +164,7 @@ public static void setServerUrl(String ustr){
}
public static String getServerUrl() {
return "http://" + serverHost + ":" + serverPort + "/" + serverWebapp + "/";
return protocol + "://" + serverHost + ":" + serverPort + "/" + serverWebapp + "/";
}
public static ResourceBundle getResourceBundle() {

19
shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
View File

@ -31,6 +31,7 @@ public class Constants {
public static final String ENV_HOST = "SQOOP2_HOST";
public static final String ENV_PORT = "SQOOP2_PORT";
public static final String ENV_WEBAPP = "SQOOP2_WEBAPP";
public static final String ENV_PROTOCOL = "SQOOP2_PROTOCOL";
// Options
@ -42,6 +43,7 @@ public class Constants {
public static final String OPT_VALUE = "value";
public static final String OPT_VERBOSE = "verbose";
public static final String OPT_HOST = "host";
public static final String OPT_TLS = "tls";
public static final String OPT_PORT = "port";
public static final String OPT_WEBAPP = "webapp";
public static final String OPT_URL = "url";
@ -59,6 +61,9 @@ public class Constants {
public static final String OPT_PRINCIPAL_TYPE = "principal-type";
public static final String OPT_WITH_GRANT = "with-grant";
public static final String OPT_WITH_JOB = "job";
public static final String OPT_TRUSTSTORE = "truststore";
public static final String OPT_TRUSTSTORE_PASSWORD = "truststore-password";
public static final String OPT_TRUSTSTORE_PASSWORD_GENERATOR = "truststore-password-generator";
public static final char OPT_FROM_CHAR = 'f';
public static final char OPT_TO_CHAR = 't';
@ -80,6 +85,7 @@ public class Constants {
public static final char OPT_ACTION_CHAR = 'a';
public static final char OPT_WITH_GRANT_CHAR = 'g';
public static final char OPT_WITH_JOB_CHAR = 'j';
public static final char OPT_TRUSTSTORE_CHAR2 = 'j';
// Resource keys for various commands, command options,
// functions and descriptions
@ -130,6 +136,7 @@ public class Constants {
public static final String FN_SUBMISSION = "submission";
public static final String FN_SERVER = "server";
public static final String FN_OPTION = "option";
public static final String FN_TRUSTSTORE = "truststore";
public static final String FN_CONNECTOR = "connector";
public static final String FN_VERSION = "version";
public static final String FN_DRIVER_CONFIG = "driver";
@ -248,12 +255,24 @@ public class Constants {
"set.webapp_description";
public static final String RES_URL_DESCRIPTION =
"set.url_description";
public static final String RES_TLS_DESCRIPTION =
"set.tls_description";
public static final String RES_SET_SERVER_USAGE =
"set.server_usage";
public static final String RES_SET_SERVER_SUCCESSFUL =
"set.server_successful";
public static final String RES_SET_SERVER_IGNORED =
"set.server_ignored";
public static final String RES_SET_TRUSTSTORE_SUCCESSFUL =
"set.truststore_successful";
public static final String RES_SET_TRUSTSTORE_FAILED=
"set.truststore_failed";
public static final String RES_TRUSTSTORE_DESCRIPTION =
"set.truststore_description";
public static final String RES_TRUSTSTORE_PASSWORD_DESCRIPTION =
"set.truststore_password_description";
public static final String RES_TRUSTSTORE_PASSWORD_GENERATOR_DESCRIPTION =
"set.truststore_password_generator_description";
public static final String RES_SHOW_PROMPT_DISPLAY_ALL_LINKS =
"show.prompt_display_all_links";

7
shell/src/main/resources/shell-resource.properties
View File

@ -121,10 +121,15 @@ set.host_description = Host name to invoke server resources
set.port_description = Port number to invoke server resources
set.webapp_description = Web app to invoke server resources
set.url_description = Url to invoke server resources
set.tls_description = Flag that determines if https should be used instead of http
set.server_usage = Usage: set server
set.server_successful = Server is set successfully
set.server_ignored = --host, --port or --webapp option is ignored, because --url option is given.
set.truststore_successful = Truststore set successfully
set.truststore_failed = Failed to set the truststore
set.truststore_description = Path to the truststore
set.truststore_password_description = Password for the truststore
set.truststore_password_generator_description = Command that prints the password to the truststore
# Show command
show.description = Display various objects and configuration options

53
test/src/test/java/org/apache/sqoop/integration/server/rest/DriverRestTest.java
View File

@ -1,53 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.integration.server.rest;
import org.apache.sqoop.test.utils.ParametrizedUtils;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Factory;
import java.util.Iterator;
public class DriverRestTest extends RestTest {
public static TestDescription[] PROVIDER_DATA = new TestDescription[]{
new TestDescription("Get driver", "v1/driver", "GET", null, new Validator() {
@Override
void validate() throws Exception {
assertResponseCode(200);
assertContains("job-config");
assertContains("all-config-resources");
}}),
new TestDescription("Invalid post request", "v1/driver", "POST", "Random data", new Validator() {
@Override
void validate() throws Exception {
assertResponseCode(405);
assertServerException("Unsupported HTTP method", "SERVER_0002");
}}),
};
@DataProvider(name="driver-rest-test")
public static Iterator<Object[]> data() {
return ParametrizedUtils.toArrayOfArrays(PROVIDER_DATA).iterator();
}
@Factory(dataProvider = "driver-rest-test")
public DriverRestTest(TestDescription desc) {
super(desc);
}
}