SQOOP-1776: Sqoop2: Delegation Token support for Authentication

(Richard Zhou via Abraham Elmahrek)

client/pom.xml

@@ -57,8 +57,7 @@ limitations under the License.
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-auth</artifactId>
-      <scope>provided</scope>
+      <artifactId>hadoop-common</artifactId>
     </dependency>
   </dependencies>
 

client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java

@@ -17,8 +17,13 @@
  */
 package org.apache.sqoop.client.request;
 
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
 import org.apache.log4j.Logger;
 import org.apache.sqoop.client.ClientError;
 import org.apache.sqoop.common.SqoopException;
@@ -34,6 +39,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
 import java.net.URL;
 import java.util.Locale;
 
@@ -42,6 +48,8 @@
  */
 public class ResourceRequest {
   private static final Logger LOG = Logger.getLogger(ResourceRequest.class);
+  private DelegationTokenAuthenticatedURL.Token authToken;
+  private String strURL;
 
   protected String doHttpRequest(String strURL, String method) {
     return doHttpRequest(strURL, method, "");
@@ -51,9 +59,10 @@ protected String doHttpRequest(String strURL, String method, String data) {
     DataOutputStream wr = null;
     BufferedReader reader = null;
     try {
-      AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+      this.authToken = new DelegationTokenAuthenticatedURL.Token();
+      this.strURL = strURL;
       URL url = new URL(strURL);
-      HttpURLConnection conn = new AuthenticatedURL().openConnection(url, token);
+      HttpURLConnection conn = new DelegationTokenAuthenticatedURL().openConnection(url, authToken);
 
       conn.setRequestMethod(method);
 //      Sqoop is using JSON for data transfers
@@ -166,4 +175,41 @@ public String put(String url, String data) {
   public String delete(String url) {
     return doHttpRequest(url, HttpMethod.DELETE);
   }
+
+  public Token<?>[] addDelegationTokens(String renewer,
+                                        Credentials credentials) throws IOException {
+    Token<?>[] tokens = null;
+    Text dtService = getDelegationTokenService();
+    Token<?> token = credentials.getToken(dtService);
+    if (token == null) {
+      URL url = new URL(strURL);
+      DelegationTokenAuthenticatedURL authUrl =
+              new DelegationTokenAuthenticatedURL(new ConnectionConfigurator() {
+                @Override
+                public HttpURLConnection configure(HttpURLConnection conn) throws IOException {
+                  return conn;
+                }
+              });
+      try {
+        token = authUrl.getDelegationToken(url, authToken, renewer);
+        if (token != null) {
+          credentials.addToken(token.getService(), token);
+          tokens = new Token<?>[]{token};
+        } else {
+          throw new IOException("Got NULL as delegation token");
+        }
+      } catch (AuthenticationException ex) {
+        throw new IOException(ex);
+      }
+    }
+    return tokens;
+  }
+
+  private Text getDelegationTokenService() throws IOException {
+    URL url = new URL(strURL);
+    InetSocketAddress addr = new InetSocketAddress(url.getHost(),
+            url.getPort());
+    Text dtService = SecurityUtil.buildTokenService(addr);
+    return dtService;
+  }
 }

common/src/main/java/org/apache/sqoop/common/MapContext.java

@@ -19,6 +19,8 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * ImmutableContext implementation based on (Hash)Map.
@@ -71,7 +73,7 @@ public boolean getBoolean(String key, boolean defaultValue) {
    */
   @Override
   public long getLong(String key, long defaultValue) {
-    if(!options.containsKey(key)) {
+    if (!options.containsKey(key)) {
       return defaultValue;
     }
 
@@ -85,7 +87,7 @@ public long getLong(String key, long defaultValue) {
    */
   @Override
   public int getInt(String key, int defaultValue) {
-    if(!options.containsKey(key)) {
+    if (!options.containsKey(key)) {
       return defaultValue;
     }
 
@@ -110,4 +112,25 @@ public Map<String, String> getNestedProperties(String prefix) {
 
     return subProps;
   }
+
+  /**
+   * get keys matching the the regex
+   *
+   * @param regex
+   * @return Map<String,String> with matching keys
+   */
+  public Map<String, String> getValByRegex(String regex) {
+    Pattern p = Pattern.compile(regex);
+
+    Map<String, String> result = new HashMap<String, String>();
+    Matcher m;
+
+    for (String item : options.keySet()) {
+      m = p.matcher(item);
+      if (m.find()) { // match
+        result.put(item, getString(item));
+      }
+    }
+    return result;
+  }
 }

core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java

@@ -96,6 +96,11 @@ public final class AuthenticationConstants {
   public static final String AUTHENTICATION_KERBEROS_HTTP_KEYTAB =
           PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG + "keytab";
 
+  /**
+   * The config specifies the token kind in delegation token.
+   */
+  public static final String TOKEN_KIND = "sqoop_token_kind";
+
   public static enum TYPE {SIMPLE, KERBEROS}
 
   private AuthenticationConstants() {

dist/src/main/server/conf/sqoop.properties

@@ -152,4 +152,8 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce
 #org.apache.sqoop.authentication.kerberos.principal=sqoop/_HOST@NOVALOCAL
 #org.apache.sqoop.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab
 #org.apache.sqoop.authentication.kerberos.http.principal=HTTP/_HOST@NOVALOCAL
-#org.apache.sqoop.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab
+#org.apache.sqoop.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab
+#org.apache.sqoop.authentication.enable.doAs=true
+#org.apache.sqoop.authentication.proxyuser.#USER#.users=*
+#org.apache.sqoop.authentication.proxyuser.#USER#.groups=*
+#org.apache.sqoop.authentication.proxyuser.#USER#.hosts=*

pom.xml

@@ -226,23 +226,6 @@ limitations under the License.
 
       <dependencyManagement>
         <dependencies>
-          <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
-            <version>${hadoop.2.version}</version>
-            <scope>provided</scope>
-            <exclusions>
-              <exclusion>
-                <groupId>org.apache.httpcomponents</groupId>
-                <artifactId>httpcore</artifactId>
-              </exclusion>
-              <exclusion>
-                <groupId>org.apache.httpcomponents</groupId>
-                <artifactId>httpclient</artifactId>
-              </exclusion>
-            </exclusions>
-          </dependency>
-
           <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-mapreduce-client-core</artifactId>
@@ -605,6 +588,11 @@ limitations under the License.
         <artifactId>jackson-databind</artifactId>
         <version>${jackson.databind.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-common</artifactId>
+        <version>${hadoop.2.version}</version>
+      </dependency>
    </dependencies>
   </dependencyManagement>
 

server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java

@@ -17,10 +17,15 @@
  */
 package org.apache.sqoop.filter;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler;
 import org.apache.sqoop.common.MapContext;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.core.SqoopConfiguration;
@@ -30,21 +35,22 @@
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
 import java.io.IOException;
+import java.util.Map;
 import java.util.Properties;
 
-public class SqoopAuthenticationFilter extends AuthenticationFilter {
+public class SqoopAuthenticationFilter extends DelegationTokenAuthenticationFilter {
 
   @Override
   protected Properties getConfiguration(String configPrefix,
                                         FilterConfig filterConfig) throws ServletException {
-    Properties properties = super.getConfiguration(configPrefix, filterConfig);
+    Properties properties = new Properties();
     MapContext mapContext = SqoopConfiguration.getInstance().getContext();
     String type = mapContext.getString(
         AuthenticationConstants.AUTHENTICATION_TYPE,
         AuthenticationConstants.TYPE.SIMPLE.name()).trim();
 
     if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.KERBEROS.name())) {
-      properties.setProperty(AUTH_TYPE, AuthenticationConstants.TYPE.KERBEROS.name().toLowerCase());
+      properties.setProperty(AUTH_TYPE, KerberosDelegationTokenAuthenticationHandler.class.getName());
 
       String keytab = mapContext.getString(
               AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB).trim();
@@ -71,13 +77,26 @@ protected Properties getConfiguration(String configPrefix,
       properties.setProperty(KerberosAuthenticationHandler.PRINCIPAL, hostPrincipal);
       properties.setProperty(KerberosAuthenticationHandler.KEYTAB, keytab);
     } else if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.SIMPLE.name())) {
-      properties.setProperty(AUTH_TYPE, PseudoAuthenticationHandler.class.getName());
+      properties.setProperty(AUTH_TYPE, PseudoDelegationTokenAuthenticationHandler.class.getName());
       properties.setProperty(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED,
           mapContext.getString(AuthenticationConstants.AUTHENTICATION_ANONYMOUS, "true").trim());
     } else {
       throw new SqoopException(AuthenticationError.AUTH_0004, type);
     }
 
+    properties.setProperty(DelegationTokenAuthenticationHandler.TOKEN_KIND,
+            AuthenticationConstants.TOKEN_KIND);
+
     return properties;
   }
-}
+
+  protected Configuration getProxyuserConfiguration(FilterConfig filterConfig) {
+    MapContext mapContext = SqoopConfiguration.getInstance().getContext();
+    Map<String, String> proxyuserConf = mapContext.getValByRegex("org\\.apache\\.sqoop\\.authentication\\.proxyuser");
+    Configuration conf = new Configuration(false);
+    for (Map.Entry<String, String> entry : proxyuserConf.entrySet()) {
+      conf.set(entry.getKey().substring("org.apache.sqoop.authentication.proxyuser.".length()), entry.getValue());
+    }
+    return conf;
+  }
+}

shell/pom.xml

@@ -70,10 +70,6 @@ limitations under the License.
       <artifactId>groovy-all</artifactId>
       <version>1.8.5</version>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-auth</artifactId>
-    </dependency>
   </dependencies>
 
   <profiles>

test/pom.xml

@@ -122,6 +122,11 @@ limitations under the License.
       <artifactId>commons-compress</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+
   </dependencies>
 
   <!-- Add classifier name to the JAR name -->
@@ -230,22 +235,6 @@ limitations under the License.
       </properties>
 
       <dependencies>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-common</artifactId>
-          <exclusions>
-            <exclusion>
-              <groupId>org.apache.httpcomponents</groupId>
-              <artifactId>httpcore</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.httpcomponents</groupId>
-              <artifactId>httpclient</artifactId>
-            </exclusion>
-          </exclusions>
-          <scope>provided</scope>
-        </dependency>
-
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-mapreduce-client-core</artifactId>

test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java

@@ -91,6 +91,8 @@ public void start() throws Exception {
          jar.contains("httpcore-")    || // Apache Http Core libraries
          jar.contains("httpclient-")  || // Apache Http Client libraries
          jar.contains("htrace-")      || // htrace-core libraries, new added in Hadoop 2.6.0
+         jar.contains("zookeeper-")   || // zookeeper libraries, new added in Hadoop 2.6.0
+         jar.contains("curator-")     || // curator libraries, new added in Hadoop 2.6.0
          jar.contains("log4j-")       || // Log4j
          jar.contains("slf4j-")       || // Slf4j
          jar.contains("jackson-")     || // Jackson