diff --git a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java index 0d407159..e2e5b67a 100644 --- a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java +++ b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java @@ -29,6 +29,7 @@ import org.apache.sqoop.model.MInput; import org.apache.sqoop.model.MInputType; import org.apache.sqoop.model.MIntegerInput; +import org.apache.sqoop.model.MListInput; import org.apache.sqoop.model.MLongInput; import org.apache.sqoop.model.MMapInput; import org.apache.sqoop.model.MStringInput; @@ -182,6 +183,10 @@ static MConfig restoreConfig(JSONObject config) { mInput = new MEnumInput(name, sensitive.booleanValue(), editable, overrides, values.split(",")); break; } + case LIST: { + mInput = new MListInput(name, sensitive.booleanValue(), editable, overrides); + break; + } default: // do nothing break; diff --git a/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java b/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java index b4146a73..5d53437c 100644 --- a/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java +++ b/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java @@ -167,6 +167,8 @@ private static MConfig toConfig(String configName, Class klass, Object object) { } else if (type.isEnum()) { input = new MEnumInput(inputName, sensitive, editable, overrides, ClassUtils.getEnumStrings(type)); + } else if (type.isAssignableFrom(List.class)) { + input = new MListInput(inputName, sensitive, editable, overrides); } else { throw new SqoopException(ModelError.MODEL_004, "Unsupported type " + type.getName() + " for input " + fieldName); diff --git a/common/src/main/java/org/apache/sqoop/model/MConfig.java b/common/src/main/java/org/apache/sqoop/model/MConfig.java index 9e129798..6f11d926 100644 --- a/common/src/main/java/org/apache/sqoop/model/MConfig.java +++ b/common/src/main/java/org/apache/sqoop/model/MConfig.java @@ -98,6 +98,10 @@ public MMapInput getMapInput(String inputName) { return (MMapInput)getInput(inputName); } + public MListInput getListInput(String inputName) { + return (MListInput)getInput(inputName); + } + @Override public String toString() { StringBuilder sb = new StringBuilder("config-").append(getName()); diff --git a/common/src/main/java/org/apache/sqoop/model/MConfigList.java b/common/src/main/java/org/apache/sqoop/model/MConfigList.java index d09434a0..2a3c3e42 100644 --- a/common/src/main/java/org/apache/sqoop/model/MConfigList.java +++ b/common/src/main/java/org/apache/sqoop/model/MConfigList.java @@ -89,6 +89,10 @@ public MBooleanInput getBooleanInput(String name) { return (MBooleanInput) getInput(name); } + public MListInput getListInput(String name) { + return (MListInput) getInput(name); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/common/src/main/java/org/apache/sqoop/model/MInputType.java b/common/src/main/java/org/apache/sqoop/model/MInputType.java index 7e6b90af..0f1fe0a6 100644 --- a/common/src/main/java/org/apache/sqoop/model/MInputType.java +++ b/common/src/main/java/org/apache/sqoop/model/MInputType.java @@ -48,5 +48,8 @@ public enum MInputType { /** Long input type */ LONG, + /** List input type */ + LIST, + ; } diff --git a/common/src/main/java/org/apache/sqoop/model/MListInput.java b/common/src/main/java/org/apache/sqoop/model/MListInput.java new file mode 100644 index 00000000..04dbaf24 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/model/MListInput.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.model; + +import java.util.LinkedList; +import java.util.List; + +import org.apache.sqoop.classification.InterfaceAudience; +import org.apache.sqoop.classification.InterfaceStability; +import org.apache.sqoop.utils.UrlSafeUtils; + +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class MListInput extends MInput> { + + public MListInput(String name, boolean sensitive, InputEditable editable, String overrides) { + super(name, sensitive, editable, overrides); + } + + @Override + public String getUrlSafeValueString() { + List valueList = getValue(); + if (valueList == null) { + return null; + } + boolean first = true; + StringBuilder vsb = new StringBuilder(); + for (String element : valueList) { + if (first) { + first = false; + } else { + vsb.append("&"); + } + vsb.append(UrlSafeUtils.urlEncode(element)); + } + return vsb.toString(); + } + + @Override + public void restoreFromUrlSafeValueString(String valueString) { + if (valueString == null) { + setValue(null); + } else { + List valueList = new LinkedList(); + if (valueString.trim().length() > 0) { + for (String element : valueString.split("&")) { + valueList.add(UrlSafeUtils.urlDecode(element)); + } + } + setValue(valueList); + } + } + + @Override + public MInputType getType() { + return MInputType.LIST; + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (!(other instanceof MListInput)) { + return false; + } + + MListInput mli = (MListInput) other; + return getName().equals(mli.getName()); + } + + @Override + public int hashCode() { + return 23 + 31 * getName().hashCode(); + } + + @Override + public boolean isEmpty() { + return getValue() == null; + } + + @Override + public void setEmpty() { + setValue(null); + } + + @Override + public Object clone(boolean cloneWithValue) { + MListInput copy = new MListInput(getName(), isSensitive(), getEditable(), getOverrides()); + copy.setPersistenceId(getPersistenceId()); + if(cloneWithValue && this.getValue() != null) { + List copyList = new LinkedList(); + for(String element : this.getValue()) { + copyList.add(element); + } + copy.setValue(copyList); + } + return copy; + } +} diff --git a/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java b/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java index d2989d3d..2a2cfe87 100644 --- a/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java +++ b/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java @@ -36,6 +36,7 @@ import org.apache.sqoop.model.MEnumInput; import org.apache.sqoop.model.MInput; import org.apache.sqoop.model.MIntegerInput; +import org.apache.sqoop.model.MListInput; import org.apache.sqoop.model.MLongInput; import org.apache.sqoop.model.MMapInput; import org.apache.sqoop.model.MStringInput; @@ -126,6 +127,9 @@ public void testInputEditableOptional() { Map map = new HashMap(); map.put("A", "B"); + List list = new LinkedList(); + list.add("C"); + // Fill config with all values MConfig config = getConfig(); config.getStringInput("String").setValue("A"); @@ -133,6 +137,7 @@ public void testInputEditableOptional() { config.getIntegerInput("Integer").setValue(1); config.getBooleanInput("Boolean").setValue(true); config.getEnumInput("Enum").setValue("YES"); + config.getListInput("List").setValue(list); // Serialize that into JSON JSONObject jsonObject = ConfigInputSerialization.extractConfig(config, MConfigType.JOB, false); @@ -168,6 +173,7 @@ public void testInputEditableOptional() { assertEquals(1, (int)retrieved.getIntegerInput("Integer").getValue()); assertEquals(true, retrieved.getBooleanInput("Boolean").getValue().booleanValue()); assertEquals("YES", retrieved.getEnumInput("Enum").getValue()); + assertEquals(list, retrieved.getListInput("List").getValue()); } protected MConfig getMapConfig() { @@ -211,6 +217,9 @@ protected MConfig getConfig() { input = new MEnumInput("Enum", false, InputEditable.ANY, StringUtils.EMPTY, new String[] {"YES", "NO"}); inputs.add(input); + input = new MListInput("List", false, InputEditable.ANY, StringUtils.EMPTY); + inputs.add(input); + return new MConfig("c", inputs); } } diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConfig.java b/common/src/test/java/org/apache/sqoop/model/TestMConfig.java index 2b47c130..2cb39899 100644 --- a/common/src/test/java/org/apache/sqoop/model/TestMConfig.java +++ b/common/src/test/java/org/apache/sqoop/model/TestMConfig.java @@ -74,6 +74,7 @@ public void testGetInputs() { StringUtils.EMPTY, (short) 3); MEnumInput enumInput = new MEnumInput("Config.D", false, InputEditable.ANY, StringUtils.EMPTY, new String[] { "I", "V" }); + MListInput listInput = new MListInput("Config.E", false, InputEditable.ANY, StringUtils.EMPTY ); List> inputs = new ArrayList>(); inputs.add(intInput); @@ -81,6 +82,7 @@ public void testGetInputs() { inputs.add(mapInput); inputs.add(stringInput); inputs.add(enumInput); + inputs.add(listInput); MConfig config = new MConfig("Config", inputs); assertEquals(intInput, config.getIntegerInput("Config.A")); @@ -88,5 +90,6 @@ public void testGetInputs() { assertEquals(mapInput, config.getMapInput("Config.B")); assertEquals(stringInput, config.getStringInput("Config.C")); assertEquals(enumInput, config.getEnumInput("Config.D")); + assertEquals(listInput, config.getListInput("Config.E")); } } diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java b/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java index 3dfc33a4..fd100b1d 100644 --- a/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java +++ b/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java @@ -43,10 +43,12 @@ public void testGetInputs() { StringUtils.EMPTY, (short) 3); MEnumInput enumInput = new MEnumInput("Config2.D", false, InputEditable.ANY, StringUtils.EMPTY, new String[] { "I", "V" }); + MListInput listInput = new MListInput("Config2.E", false, InputEditable.ANY, StringUtils.EMPTY); inputs = new ArrayList>(); inputs.add(stringInput); inputs.add(enumInput); + inputs.add(listInput); configs.add(new MConfig("Config2", inputs)); MConfigList config = new MConfigList(configs, MConfigType.JOB); @@ -54,5 +56,6 @@ public void testGetInputs() { assertEquals(mapInput, config.getMapInput("Config1.B")); assertEquals(stringInput, config.getStringInput("Config2.C")); assertEquals(enumInput, config.getEnumInput("Config2.D")); + assertEquals(listInput, config.getListInput("Config2.E")); } } diff --git a/common/src/test/java/org/apache/sqoop/model/TestMListInput.java b/common/src/test/java/org/apache/sqoop/model/TestMListInput.java new file mode 100644 index 00000000..decf9273 --- /dev/null +++ b/common/src/test/java/org/apache/sqoop/model/TestMListInput.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.model; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.testng.annotations.Test; + +/** + * Test class for org.apache.sqoop.model.MListInput + */ +public class TestMListInput { + /** + * Test for class initialization + */ + @Test + public void testInitialization() { + MListInput input = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); + assertEquals("sqoopsqoop", input.getName()); + assertEquals(MInputType.LIST, input.getType()); + } + + /** + * Test for equals() method + */ + @Test + public void testEquals() { + // Positive test + MListInput input1 = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); + MListInput input2 = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); + assertTrue(input1.equals(input2)); + + // Negative test + MListInput input3 = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); + MListInput input4 = new MListInput("sqoopsqoop1", false, InputEditable.ANY, StringUtils.EMPTY); + assertFalse(input3.equals(input4)); + } + + /** + * Test for value + */ + @Test + public void testValue() { + MListInput input1 = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); + List list = new LinkedList(); + input1.setValue(list); + assertEquals(list, input1.getValue()); + input1.setEmpty(); + assertNull(input1.getValue()); + } + + /** + * Test for getUrlSafeValueString() and restoreFromUrlSafeValueString() + */ + @Test + public void testUrlSafe() { + MListInput input1 = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY); + List list = new LinkedList(); + input1.setValue(list); + // Getting URL safe string + String tmp = input1.getUrlSafeValueString(); + // Restore to actual value + input1.restoreFromUrlSafeValueString(tmp); + assertNotNull(input1.getValue()); + assertEquals(0, input1.getValue().size()); + + input1.setValue(null); + tmp = input1.getUrlSafeValueString(); + input1.restoreFromUrlSafeValueString(tmp); + assertNull(input1.getValue()); + } + + /** + * Test case for MNamedElement.getLabelKey() and MNamedElement.getHelpKey() + */ + @Test + public void testNamedElement() { + MListInput input1 = new MListInput("sqoopsqoop", true, InputEditable.ANY, StringUtils.EMPTY); + assertEquals("sqoopsqoop.label", input1.getLabelKey()); + assertEquals("sqoopsqoop.help", input1.getHelpKey()); + } + + /** + * Test for sensitivity + */ + @Test + public void testSensitivity() { + MListInput input1 = new MListInput("NAME", false, InputEditable.ANY, StringUtils.EMPTY ); + MListInput input2 = new MListInput("NAME", true, InputEditable.ANY, StringUtils.EMPTY ); + assertFalse(input1.isSensitive()); + assertTrue(input2.isSensitive()); + } +} diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java index 46235be2..6d303c82 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java @@ -59,6 +59,7 @@ import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MLinkConfig; +import org.apache.sqoop.model.MListInput; import org.apache.sqoop.model.MLongInput; import org.apache.sqoop.model.MMapInput; import org.apache.sqoop.model.MStringInput; @@ -1978,6 +1979,9 @@ private void loadDriverConfigs(List driverConfig, case ENUM: input = new MEnumInput(inputName, inputSensitivity, editableEnum, overrides, inputEnumValues.split(",")); break; + case LIST: + input = new MListInput(inputName, inputSensitivity, editableEnum, overrides); + break; default: throw new SqoopException(CommonRepositoryError.COMMON_0003, "input-" + inputName + ":" + inputId + ":" @@ -2121,6 +2125,9 @@ public void loadConnectorConfigs(List linkConfig, List fromCon input = new MEnumInput(inputName, inputSensitivity, editableEnum, overrides, inputEnumValues.split(",")); break; + case LIST: + input = new MListInput(inputName, inputSensitivity, editableEnum, overrides); + break; default: throw new SqoopException(CommonRepositoryError.COMMON_0003, "input-" + inputName + ":" + inputId + ":" + "config-" + inputConfig + ":" + mit.name()); diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java index 6540a529..ab442fe8 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java +++ b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java @@ -40,6 +40,7 @@ import org.apache.sqoop.model.MIntegerInput; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; +import org.apache.sqoop.model.MListInput; import org.apache.sqoop.model.MLongInput; import org.apache.sqoop.model.MMapInput; import org.apache.sqoop.model.MStringInput; @@ -202,6 +203,9 @@ private static void displayConfig(MConfig config, ResourceBundle bundle) { case ENUM: displayInputEnum((MEnumInput) input); break; + case LIST: + displayInputList((MListInput) input); + break; default: print("\n%s " + input.getType(), resourceString(Constants.RES_CONFIG_DISPLAYER_UNSUPPORTED_DATATYPE)); return; @@ -272,6 +276,19 @@ private static void displayInputEnum(MEnumInput input) { print(input.getValue()); } + /** + * Display content of List input + * + * @param input List input + */ + private static void displayInputList(MListInput input) { + for (String element : input.getValue()) { + println(); + print(" "); + print(element); + } + } + private ConfigDisplayer() { // Do not instantiate } diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java index 8b5f380b..b45d8df6 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java +++ b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java @@ -29,6 +29,7 @@ import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MInput; import org.apache.sqoop.model.MIntegerInput; +import org.apache.sqoop.model.MListInput; import org.apache.sqoop.model.MLongInput; import org.apache.sqoop.model.MMapInput; import org.apache.sqoop.model.MJob; @@ -39,6 +40,7 @@ import org.apache.sqoop.validation.Status; import java.io.IOException; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.HashMap; @@ -197,12 +199,44 @@ public static boolean fillInput(String prefix, MInput input, CommandLine line) t case ENUM: assert input instanceof MEnumInput; return fillInputEnum(prefix, (MEnumInput) input, line); + case LIST: + assert input instanceof MListInput; + return fillInputList(prefix, (MListInput) input, line); default: println("Unsupported data type " + input.getType()); return true; } } + /** + * Load CLI options for list type. + * + * Parses elements that take the config "&&...". + * + * @param prefix placed at the beginning of the CLI option key + * @param input Input that we should read or edit + * @param line CLI options container + * @return + * @throws IOException + */ + private static boolean fillInputList(String prefix, + MListInput input, + CommandLine line) + throws IOException { + String opt = ConfigOptions.getOptionKey(prefix, input); + if (line.hasOption(opt)) { + String value = line.getOptionValue(opt); + List values = new LinkedList(); + for (String element : value.split("&")) { + values.add(element); + } + input.setValue(values); + } else { + input.setEmpty(); + } + return true; + } + /** * Load CLI option for enum type. * @@ -503,12 +537,80 @@ static boolean fillInputWithBundle(MInput input, ConsoleReader reader, ResourceB return fillInputMapWithBundle((MMapInput) input, reader, bundle); case ENUM: return fillInputEnumWithBundle((MEnumInput) input, reader, bundle); + case LIST: + return fillInputListWithBundle((MListInput) input, reader, bundle); default: println("Unsupported data type " + input.getType()); return true; } } + /** + * Load user input for list type. + * + * This implementation will load one element at a time. If user did not + * enter anything (empty input) finish loading and return from function. + * + * @param input Input that we should read or edit + * @param reader Associated console reader + * @param bundle Resource bundle + * @return True if user wish to continue with loading additional inputs + * @throws IOException + */ + private static boolean fillInputListWithBundle(MListInput input, + ConsoleReader reader, + ResourceBundle bundle) + throws IOException { + // Special prompt in List case + println(bundle.getString(input.getLabelKey()) + ": "); + + // Internal loading list + List values = input.getValue(); + if(values == null) { + values = new LinkedList(); + } + + String userTyped; + + while(true) { + // Print all current items in each iteration + // However do not printout if this input contains sensitive information. + println("There are currently " + values.size() + " values in the list:"); + if (!input.isSensitive()) { + for(String value : values) { + println(value); + } + } + + // Special prompt for List element + reader.printString("element# "); + reader.flushConsole(); + + if(input.isSensitive()) { + userTyped = reader.readLine('*'); + } else { + userTyped = reader.readLine(); + } + + if(userTyped == null) { + // Finish loading and return back to Sqoop shell + return false; + } else if(userTyped.isEmpty()) { + // User has finished loading data to List input, either set input empty + // if there are no elements or propagate elements to the input + if(values.size() == 0) { + input.setEmpty(); + } else { + input.setValue(values); + } + return true; + } else { + values.add(userTyped); + } + + } + } + /** * Load user input for enum type. *