From 77be2a8f9c8be384ae453b3f26296d30d4a40893 Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Thu, 15 Oct 2015 16:53:07 -0700 Subject: [PATCH] SQOOP-2618: Sqoop2: Mask S3 sensitive properties in HDFS configuration (Abraham Fine via Jarek Jarcec Cecho) --- .../hdfs/configuration/LinkConfig.java | 3 +- .../sqoop/connector/hdfs/TestLinkConfig.java | 42 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java index 39f3752a..5852e538 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java @@ -36,7 +36,8 @@ public class LinkConfig { @Input(size = 255, validators = { @Validator(DirectoryExistsValidator.class)}) public String confDir; - @Input public Map configOverrides; + @Input(sensitiveKeyPattern = "(?i)(.*(password|key|secret).*)") + public Map configOverrides; public LinkConfig() { configOverrides = new HashMap<>(); diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java index 05178acc..900e3db9 100644 --- a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java +++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java @@ -18,8 +18,15 @@ package org.apache.sqoop.connector.hdfs; import org.apache.sqoop.connector.hdfs.configuration.LinkConfig; +import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration; +import org.apache.sqoop.model.ConfigUtils; +import org.apache.sqoop.model.MConfig; +import org.apache.sqoop.model.MMapInput; import org.testng.annotations.Test; +import java.util.Map; + +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -68,4 +75,39 @@ public void testInvalidURI() { assertFalse(validator.getStatus().canProceed(), uri); } } + + @Test + public void testSensitiveConfigOverridesKeys() { + String nonSensitiveKey = "public"; + String valueString = "value"; + String filler = "blah"; + + String[] sensitiveWords = new String[] {"password", "key", "secret"}; + + LinkConfiguration linkConfiguration = new LinkConfiguration(); + LinkConfig config = new LinkConfig(); + linkConfiguration.linkConfig = config; + config.configOverrides.put(nonSensitiveKey, valueString); + for (String sensitiveWord : sensitiveWords) { + for (String sensitiveWordWithCase : new String[] {sensitiveWord, sensitiveWord.toUpperCase()}) { + config.configOverrides.put(filler + sensitiveWordWithCase + filler, valueString); + config.configOverrides.put(sensitiveWordWithCase + filler, valueString); + config.configOverrides.put(filler + sensitiveWordWithCase, valueString); + } + } + + MConfig mLinkConfig = ConfigUtils.toConfigs(linkConfiguration).get(0); + MMapInput mConfigOverrides = mLinkConfig.getMapInput("linkConfig.configOverrides"); + + Map redactedMap = mConfigOverrides.getNonsenstiveValue(); + assertEquals(redactedMap.get(nonSensitiveKey), valueString); + + for (String sensitiveWord : sensitiveWords) { + for (String sensitiveWordWithCase : new String[] {sensitiveWord, sensitiveWord.toUpperCase()}) { + assertEquals(redactedMap.get(filler + sensitiveWordWithCase + filler), MMapInput.SENSITIVE_VALUE_PLACEHOLDER); + assertEquals(redactedMap.get(sensitiveWordWithCase + filler), MMapInput.SENSITIVE_VALUE_PLACEHOLDER); + assertEquals(redactedMap.get(filler + sensitiveWordWithCase), MMapInput.SENSITIVE_VALUE_PLACEHOLDER); + } + } + } }