From c89f168d494ee9a73b784dc9bab0d8465ccc97e4 Mon Sep 17 00:00:00 2001 From: Abraham Elmahrek Date: Sun, 28 Dec 2014 00:22:52 -0800 Subject: [PATCH] SQOOP-1776: Sqoop2: Delegation Token support for Authentication (Richard Zhou via Abraham Elmahrek) --- client/pom.xml | 3 +- .../sqoop/client/request/ResourceRequest.java | 52 +++++++++++++++++-- .../org/apache/sqoop/common/MapContext.java | 27 +++++++++- .../security/AuthenticationConstants.java | 5 ++ dist/src/main/server/conf/sqoop.properties | 6 ++- pom.xml | 22 ++------ .../filter/SqoopAuthenticationFilter.java | 29 +++++++++-- shell/pom.xml | 4 -- test/pom.xml | 21 ++------ .../minicluster/TomcatSqoopMiniCluster.java | 2 + 10 files changed, 121 insertions(+), 50 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index b53c9d3d..d424e8d9 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -57,8 +57,7 @@ limitations under the License. org.apache.hadoop - hadoop-auth - provided + hadoop-common diff --git a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java index 5f134d58..e55ba360 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java +++ b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java @@ -17,8 +17,13 @@ */ package org.apache.sqoop.client.request; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; import org.apache.log4j.Logger; import org.apache.sqoop.client.ClientError; import org.apache.sqoop.common.SqoopException; @@ -34,6 +39,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.HttpURLConnection; +import java.net.InetSocketAddress; import java.net.URL; import java.util.Locale; @@ -42,6 +48,8 @@ */ public class ResourceRequest { private static final Logger LOG = Logger.getLogger(ResourceRequest.class); + private DelegationTokenAuthenticatedURL.Token authToken; + private String strURL; protected String doHttpRequest(String strURL, String method) { return doHttpRequest(strURL, method, ""); @@ -51,9 +59,10 @@ protected String doHttpRequest(String strURL, String method, String data) { DataOutputStream wr = null; BufferedReader reader = null; try { - AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + this.authToken = new DelegationTokenAuthenticatedURL.Token(); + this.strURL = strURL; URL url = new URL(strURL); - HttpURLConnection conn = new AuthenticatedURL().openConnection(url, token); + HttpURLConnection conn = new DelegationTokenAuthenticatedURL().openConnection(url, authToken); conn.setRequestMethod(method); // Sqoop is using JSON for data transfers @@ -166,4 +175,41 @@ public String put(String url, String data) { public String delete(String url) { return doHttpRequest(url, HttpMethod.DELETE); } + + public Token[] addDelegationTokens(String renewer, + Credentials credentials) throws IOException { + Token[] tokens = null; + Text dtService = getDelegationTokenService(); + Token token = credentials.getToken(dtService); + if (token == null) { + URL url = new URL(strURL); + DelegationTokenAuthenticatedURL authUrl = + new DelegationTokenAuthenticatedURL(new ConnectionConfigurator() { + @Override + public HttpURLConnection configure(HttpURLConnection conn) throws IOException { + return conn; + } + }); + try { + token = authUrl.getDelegationToken(url, authToken, renewer); + if (token != null) { + credentials.addToken(token.getService(), token); + tokens = new Token[]{token}; + } else { + throw new IOException("Got NULL as delegation token"); + } + } catch (AuthenticationException ex) { + throw new IOException(ex); + } + } + return tokens; + } + + private Text getDelegationTokenService() throws IOException { + URL url = new URL(strURL); + InetSocketAddress addr = new InetSocketAddress(url.getHost(), + url.getPort()); + Text dtService = SecurityUtil.buildTokenService(addr); + return dtService; + } } diff --git a/common/src/main/java/org/apache/sqoop/common/MapContext.java b/common/src/main/java/org/apache/sqoop/common/MapContext.java index 2229889c..d1f9420e 100644 --- a/common/src/main/java/org/apache/sqoop/common/MapContext.java +++ b/common/src/main/java/org/apache/sqoop/common/MapContext.java @@ -19,6 +19,8 @@ import java.util.HashMap; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * ImmutableContext implementation based on (Hash)Map. @@ -71,7 +73,7 @@ public boolean getBoolean(String key, boolean defaultValue) { */ @Override public long getLong(String key, long defaultValue) { - if(!options.containsKey(key)) { + if (!options.containsKey(key)) { return defaultValue; } @@ -85,7 +87,7 @@ public long getLong(String key, long defaultValue) { */ @Override public int getInt(String key, int defaultValue) { - if(!options.containsKey(key)) { + if (!options.containsKey(key)) { return defaultValue; } @@ -110,4 +112,25 @@ public Map getNestedProperties(String prefix) { return subProps; } + + /** + * get keys matching the the regex + * + * @param regex + * @return Map with matching keys + */ + public Map getValByRegex(String regex) { + Pattern p = Pattern.compile(regex); + + Map result = new HashMap(); + Matcher m; + + for (String item : options.keySet()) { + m = p.matcher(item); + if (m.find()) { // match + result.put(item, getString(item)); + } + } + return result; + } } diff --git a/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java b/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java index f37e5058..ec2f32d5 100644 --- a/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java +++ b/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java @@ -96,6 +96,11 @@ public final class AuthenticationConstants { public static final String AUTHENTICATION_KERBEROS_HTTP_KEYTAB = PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG + "keytab"; + /** + * The config specifies the token kind in delegation token. + */ + public static final String TOKEN_KIND = "sqoop_token_kind"; + public static enum TYPE {SIMPLE, KERBEROS} private AuthenticationConstants() { diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties index fe4dfb62..2ae1aba6 100755 --- a/dist/src/main/server/conf/sqoop.properties +++ b/dist/src/main/server/conf/sqoop.properties @@ -152,4 +152,8 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce #org.apache.sqoop.authentication.kerberos.principal=sqoop/_HOST@NOVALOCAL #org.apache.sqoop.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab #org.apache.sqoop.authentication.kerberos.http.principal=HTTP/_HOST@NOVALOCAL -#org.apache.sqoop.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab \ No newline at end of file +#org.apache.sqoop.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab +#org.apache.sqoop.authentication.enable.doAs=true +#org.apache.sqoop.authentication.proxyuser.#USER#.users=* +#org.apache.sqoop.authentication.proxyuser.#USER#.groups=* +#org.apache.sqoop.authentication.proxyuser.#USER#.hosts=* \ No newline at end of file diff --git a/pom.xml b/pom.xml index c7c9d78c..ea157f79 100644 --- a/pom.xml +++ b/pom.xml @@ -226,23 +226,6 @@ limitations under the License. - - org.apache.hadoop - hadoop-common - ${hadoop.2.version} - provided - - - org.apache.httpcomponents - httpcore - - - org.apache.httpcomponents - httpclient - - - - org.apache.hadoop hadoop-mapreduce-client-core @@ -605,6 +588,11 @@ limitations under the License. jackson-databind ${jackson.databind.version} + + org.apache.hadoop + hadoop-common + ${hadoop.2.version} + diff --git a/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java b/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java index bb662904..2b6ef348 100644 --- a/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java +++ b/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java @@ -17,10 +17,15 @@ */ package org.apache.sqoop.filter; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; import org.apache.sqoop.common.MapContext; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.core.SqoopConfiguration; @@ -30,21 +35,22 @@ import javax.servlet.FilterConfig; import javax.servlet.ServletException; import java.io.IOException; +import java.util.Map; import java.util.Properties; -public class SqoopAuthenticationFilter extends AuthenticationFilter { +public class SqoopAuthenticationFilter extends DelegationTokenAuthenticationFilter { @Override protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException { - Properties properties = super.getConfiguration(configPrefix, filterConfig); + Properties properties = new Properties(); MapContext mapContext = SqoopConfiguration.getInstance().getContext(); String type = mapContext.getString( AuthenticationConstants.AUTHENTICATION_TYPE, AuthenticationConstants.TYPE.SIMPLE.name()).trim(); if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.KERBEROS.name())) { - properties.setProperty(AUTH_TYPE, AuthenticationConstants.TYPE.KERBEROS.name().toLowerCase()); + properties.setProperty(AUTH_TYPE, KerberosDelegationTokenAuthenticationHandler.class.getName()); String keytab = mapContext.getString( AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB).trim(); @@ -71,13 +77,26 @@ protected Properties getConfiguration(String configPrefix, properties.setProperty(KerberosAuthenticationHandler.PRINCIPAL, hostPrincipal); properties.setProperty(KerberosAuthenticationHandler.KEYTAB, keytab); } else if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.SIMPLE.name())) { - properties.setProperty(AUTH_TYPE, PseudoAuthenticationHandler.class.getName()); + properties.setProperty(AUTH_TYPE, PseudoDelegationTokenAuthenticationHandler.class.getName()); properties.setProperty(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, mapContext.getString(AuthenticationConstants.AUTHENTICATION_ANONYMOUS, "true").trim()); } else { throw new SqoopException(AuthenticationError.AUTH_0004, type); } + properties.setProperty(DelegationTokenAuthenticationHandler.TOKEN_KIND, + AuthenticationConstants.TOKEN_KIND); + return properties; } -} + + protected Configuration getProxyuserConfiguration(FilterConfig filterConfig) { + MapContext mapContext = SqoopConfiguration.getInstance().getContext(); + Map proxyuserConf = mapContext.getValByRegex("org\\.apache\\.sqoop\\.authentication\\.proxyuser"); + Configuration conf = new Configuration(false); + for (Map.Entry entry : proxyuserConf.entrySet()) { + conf.set(entry.getKey().substring("org.apache.sqoop.authentication.proxyuser.".length()), entry.getValue()); + } + return conf; + } +} \ No newline at end of file diff --git a/shell/pom.xml b/shell/pom.xml index 2092690c..e48c86a2 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -70,10 +70,6 @@ limitations under the License. groovy-all 1.8.5 - - org.apache.hadoop - hadoop-auth - diff --git a/test/pom.xml b/test/pom.xml index 38b22798..f74ee3c9 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -122,6 +122,11 @@ limitations under the License. commons-compress + + org.apache.hadoop + hadoop-common + + @@ -230,22 +235,6 @@ limitations under the License. - - org.apache.hadoop - hadoop-common - - - org.apache.httpcomponents - httpcore - - - org.apache.httpcomponents - httpclient - - - provided - - org.apache.hadoop hadoop-mapreduce-client-core diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java index 1e6706ef..9c099fc6 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java @@ -91,6 +91,8 @@ public void start() throws Exception { jar.contains("httpcore-") || // Apache Http Core libraries jar.contains("httpclient-") || // Apache Http Client libraries jar.contains("htrace-") || // htrace-core libraries, new added in Hadoop 2.6.0 + jar.contains("zookeeper-") || // zookeeper libraries, new added in Hadoop 2.6.0 + jar.contains("curator-") || // curator libraries, new added in Hadoop 2.6.0 jar.contains("log4j-") || // Log4j jar.contains("slf4j-") || // Slf4j jar.contains("jackson-") || // Jackson