SQOOP-2462: Sqoop2: Provide List type for configuration objects

(Dian Fu via Jarek Jarcec Cecho)

common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java

@@ -29,6 +29,7 @@
 import org.apache.sqoop.model.MInput;
 import org.apache.sqoop.model.MInputType;
 import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MListInput;
 import org.apache.sqoop.model.MLongInput;
 import org.apache.sqoop.model.MMapInput;
 import org.apache.sqoop.model.MStringInput;
@@ -182,6 +183,10 @@ static MConfig restoreConfig(JSONObject config) {
         mInput = new MEnumInput(name, sensitive.booleanValue(), editable, overrides, values.split(","));
         break;
       }
+      case LIST: {
+        mInput = new MListInput(name, sensitive.booleanValue(), editable, overrides);
+        break;
+      }
       default:
         // do nothing
         break;

common/src/main/java/org/apache/sqoop/model/ConfigUtils.java

@@ -167,6 +167,8 @@ private static MConfig toConfig(String configName, Class klass, Object object) {
         } else if (type.isEnum()) {
           input = new MEnumInput(inputName, sensitive, editable, overrides,
               ClassUtils.getEnumStrings(type));
+        } else if (type.isAssignableFrom(List.class)) {
+          input = new MListInput(inputName, sensitive, editable, overrides);
         } else {
           throw new SqoopException(ModelError.MODEL_004, "Unsupported type "
               + type.getName() + " for input " + fieldName);

common/src/main/java/org/apache/sqoop/model/MConfig.java

@@ -98,6 +98,10 @@ public MMapInput getMapInput(String inputName) {
     return (MMapInput)getInput(inputName);
   }
 
+  public MListInput getListInput(String inputName) {
+    return (MListInput)getInput(inputName);
+  }
+
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder("config-").append(getName());

common/src/main/java/org/apache/sqoop/model/MConfigList.java

@@ -89,6 +89,10 @@ public MBooleanInput getBooleanInput(String name) {
     return (MBooleanInput) getInput(name);
   }
 
+  public MListInput getListInput(String name) {
+    return (MListInput) getInput(name);
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {

common/src/main/java/org/apache/sqoop/model/MInputType.java

@@ -48,5 +48,8 @@ public enum MInputType {
   /** Long input type */
   LONG,
 
+  /** List input type */
+  LIST,
+
   ;
 }

common/src/main/java/org/apache/sqoop/model/MListInput.java

@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.sqoop.classification.InterfaceAudience;
+import org.apache.sqoop.classification.InterfaceStability;
+import org.apache.sqoop.utils.UrlSafeUtils;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class MListInput extends MInput<List<String>> {
+
+  public MListInput(String name, boolean sensitive, InputEditable editable, String overrides) {
+    super(name, sensitive, editable, overrides);
+  }
+
+  @Override
+  public String getUrlSafeValueString() {
+    List<String> valueList = getValue();
+    if (valueList == null) {
+      return null;
+    }
+    boolean first = true;
+    StringBuilder vsb = new StringBuilder();
+    for (String element : valueList) {
+      if (first) {
+        first = false;
+      } else {
+        vsb.append("&");
+      }
+      vsb.append(UrlSafeUtils.urlEncode(element));
+    }
+    return vsb.toString();
+  }
+
+  @Override
+  public void restoreFromUrlSafeValueString(String valueString) {
+    if (valueString == null) {
+      setValue(null);
+    } else {
+      List<String> valueList = new LinkedList<String>();
+      if (valueString.trim().length() > 0) {
+        for (String element : valueString.split("&")) {
+          valueList.add(UrlSafeUtils.urlDecode(element));
+        }
+      }
+      setValue(valueList);
+    }
+  }
+
+  @Override
+  public MInputType getType() {
+    return MInputType.LIST;
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == this) {
+      return true;
+    }
+
+    if (!(other instanceof MListInput)) {
+      return false;
+    }
+
+    MListInput mli = (MListInput) other;
+    return getName().equals(mli.getName());
+  }
+
+  @Override
+  public int hashCode() {
+    return 23 + 31 * getName().hashCode();
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return getValue() == null;
+  }
+
+  @Override
+  public void setEmpty() {
+    setValue(null);
+  }
+
+  @Override
+  public Object clone(boolean cloneWithValue) {
+    MListInput copy = new MListInput(getName(), isSensitive(), getEditable(), getOverrides());
+    copy.setPersistenceId(getPersistenceId());
+    if(cloneWithValue && this.getValue() != null) {
+      List<String> copyList = new LinkedList<String>();
+      for(String element : this.getValue()) {
+        copyList.add(element);
+      }
+      copy.setValue(copyList);
+    }
+    return copy;
+  }
+}

common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java

@@ -36,6 +36,7 @@
 import org.apache.sqoop.model.MEnumInput;
 import org.apache.sqoop.model.MInput;
 import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MListInput;
 import org.apache.sqoop.model.MLongInput;
 import org.apache.sqoop.model.MMapInput;
 import org.apache.sqoop.model.MStringInput;
@@ -126,6 +127,9 @@ public void testInputEditableOptional() {
     Map<String, String> map = new HashMap<String, String>();
     map.put("A", "B");
 
+    List<String> list = new LinkedList<String>();
+    list.add("C");
+
     // Fill config with all values
     MConfig config = getConfig();
     config.getStringInput("String").setValue("A");
@@ -133,6 +137,7 @@ public void testInputEditableOptional() {
     config.getIntegerInput("Integer").setValue(1);
     config.getBooleanInput("Boolean").setValue(true);
     config.getEnumInput("Enum").setValue("YES");
+    config.getListInput("List").setValue(list);
 
     // Serialize that into JSON
     JSONObject jsonObject = ConfigInputSerialization.extractConfig(config, MConfigType.JOB,  false);
@@ -168,6 +173,7 @@ public void testInputEditableOptional() {
     assertEquals(1, (int)retrieved.getIntegerInput("Integer").getValue());
     assertEquals(true, retrieved.getBooleanInput("Boolean").getValue().booleanValue());
     assertEquals("YES", retrieved.getEnumInput("Enum").getValue());
+    assertEquals(list, retrieved.getListInput("List").getValue());
   }
 
   protected MConfig getMapConfig() {
@@ -211,6 +217,9 @@ protected MConfig getConfig() {
     input = new MEnumInput("Enum", false, InputEditable.ANY, StringUtils.EMPTY, new String[] {"YES", "NO"});
     inputs.add(input);
 
+    input = new MListInput("List", false, InputEditable.ANY, StringUtils.EMPTY);
+    inputs.add(input);
+
     return new MConfig("c", inputs);
   }
 }

common/src/test/java/org/apache/sqoop/model/TestMConfig.java

@@ -74,6 +74,7 @@ public void testGetInputs() {
         StringUtils.EMPTY, (short) 3);
     MEnumInput enumInput = new MEnumInput("Config.D", false, InputEditable.ANY, StringUtils.EMPTY,
         new String[] { "I", "V" });
+    MListInput listInput = new MListInput("Config.E", false, InputEditable.ANY, StringUtils.EMPTY );
 
     List<MInput<?>> inputs = new ArrayList<MInput<?>>();
     inputs.add(intInput);
@@ -81,6 +82,7 @@ public void testGetInputs() {
     inputs.add(mapInput);
     inputs.add(stringInput);
     inputs.add(enumInput);
+    inputs.add(listInput);
 
     MConfig config = new MConfig("Config", inputs);
     assertEquals(intInput, config.getIntegerInput("Config.A"));
@@ -88,5 +90,6 @@ public void testGetInputs() {
     assertEquals(mapInput, config.getMapInput("Config.B"));
     assertEquals(stringInput, config.getStringInput("Config.C"));
     assertEquals(enumInput, config.getEnumInput("Config.D"));
+    assertEquals(listInput, config.getListInput("Config.E"));
   }
 }

common/src/test/java/org/apache/sqoop/model/TestMConfigList.java

@@ -43,10 +43,12 @@ public void testGetInputs() {
         StringUtils.EMPTY, (short) 3);
     MEnumInput enumInput = new MEnumInput("Config2.D", false, InputEditable.ANY, StringUtils.EMPTY,
         new String[] { "I", "V" });
+    MListInput listInput = new MListInput("Config2.E", false, InputEditable.ANY, StringUtils.EMPTY);
 
     inputs = new ArrayList<MInput<?>>();
     inputs.add(stringInput);
     inputs.add(enumInput);
+    inputs.add(listInput);
     configs.add(new MConfig("Config2", inputs));
 
     MConfigList config = new MConfigList(configs, MConfigType.JOB);
@@ -54,5 +56,6 @@ public void testGetInputs() {
     assertEquals(mapInput, config.getMapInput("Config1.B"));
     assertEquals(stringInput, config.getStringInput("Config2.C"));
     assertEquals(enumInput, config.getEnumInput("Config2.D"));
+    assertEquals(listInput, config.getListInput("Config2.E"));
   }
 }

common/src/test/java/org/apache/sqoop/model/TestMListInput.java

@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.testng.annotations.Test;
+
+/**
+ * Test class for org.apache.sqoop.model.MListInput
+ */
+public class TestMListInput {
+  /**
+   * Test for class initialization
+   */
+  @Test
+  public void testInitialization() {
+    MListInput input = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
+    assertEquals("sqoopsqoop", input.getName());
+    assertEquals(MInputType.LIST, input.getType());
+  }
+
+  /**
+   * Test for equals() method
+   */
+  @Test
+  public void testEquals() {
+    // Positive test
+    MListInput input1 = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
+    MListInput input2 = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
+    assertTrue(input1.equals(input2));
+
+    // Negative test
+    MListInput input3 = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
+    MListInput input4 = new MListInput("sqoopsqoop1", false, InputEditable.ANY, StringUtils.EMPTY);
+    assertFalse(input3.equals(input4));
+  }
+
+  /**
+   * Test for value
+   */
+  @Test
+  public void testValue() {
+    MListInput input1 = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
+    List<String> list = new LinkedList<String>();
+    input1.setValue(list);
+    assertEquals(list, input1.getValue());
+    input1.setEmpty();
+    assertNull(input1.getValue());
+  }
+
+  /**
+   * Test for getUrlSafeValueString() and restoreFromUrlSafeValueString()
+   */
+  @Test
+  public void testUrlSafe() {
+    MListInput input1 = new MListInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
+    List<String> list = new LinkedList<String>();
+    input1.setValue(list);
+    // Getting URL safe string
+    String tmp = input1.getUrlSafeValueString();
+    // Restore to actual value
+    input1.restoreFromUrlSafeValueString(tmp);
+    assertNotNull(input1.getValue());
+    assertEquals(0, input1.getValue().size());
+
+    input1.setValue(null);
+    tmp = input1.getUrlSafeValueString();
+    input1.restoreFromUrlSafeValueString(tmp);
+    assertNull(input1.getValue());
+  }
+
+  /**
+   * Test case for MNamedElement.getLabelKey() and MNamedElement.getHelpKey()
+   */
+  @Test
+  public void testNamedElement() {
+    MListInput input1 = new MListInput("sqoopsqoop", true, InputEditable.ANY, StringUtils.EMPTY);
+    assertEquals("sqoopsqoop.label", input1.getLabelKey());
+    assertEquals("sqoopsqoop.help", input1.getHelpKey());
+  }
+
+  /**
+   * Test for sensitivity
+   */
+  @Test
+  public void testSensitivity() {
+    MListInput input1 = new MListInput("NAME", false, InputEditable.ANY, StringUtils.EMPTY );
+    MListInput input2 = new MListInput("NAME", true, InputEditable.ANY, StringUtils.EMPTY );
+    assertFalse(input1.isSensitive());
+    assertTrue(input2.isSensitive());
+  }
+}

repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java

@@ -59,6 +59,7 @@
 import org.apache.sqoop.model.MJob;
 import org.apache.sqoop.model.MLink;
 import org.apache.sqoop.model.MLinkConfig;
+import org.apache.sqoop.model.MListInput;
 import org.apache.sqoop.model.MLongInput;
 import org.apache.sqoop.model.MMapInput;
 import org.apache.sqoop.model.MStringInput;
@@ -1978,6 +1979,9 @@ private void loadDriverConfigs(List<MConfig> driverConfig,
               case ENUM:
                 input = new MEnumInput(inputName, inputSensitivity, editableEnum, overrides, inputEnumValues.split(","));
                 break;
+              case LIST:
+                input = new MListInput(inputName, inputSensitivity, editableEnum, overrides);
+                break;
               default:
                 throw new SqoopException(CommonRepositoryError.COMMON_0003,
                         "input-" + inputName + ":" + inputId + ":"
@@ -2121,6 +2125,9 @@ public void loadConnectorConfigs(List<MConfig> linkConfig, List<MConfig> fromCon
                 input = new MEnumInput(inputName, inputSensitivity, editableEnum, overrides,
                         inputEnumValues.split(","));
                 break;
+              case LIST:
+                input = new MListInput(inputName, inputSensitivity, editableEnum, overrides);
+                break;
               default:
                 throw new SqoopException(CommonRepositoryError.COMMON_0003, "input-" + inputName + ":"
                         + inputId + ":" + "config-" + inputConfig + ":" + mit.name());

shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java

@@ -40,6 +40,7 @@
 import org.apache.sqoop.model.MIntegerInput;
 import org.apache.sqoop.model.MJob;
 import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MListInput;
 import org.apache.sqoop.model.MLongInput;
 import org.apache.sqoop.model.MMapInput;
 import org.apache.sqoop.model.MStringInput;
@@ -202,6 +203,9 @@ private static void displayConfig(MConfig config, ResourceBundle bundle) {
             case ENUM:
               displayInputEnum((MEnumInput) input);
               break;
+            case LIST:
+              displayInputList((MListInput) input);
+              break;
             default:
               print("\n%s " + input.getType(), resourceString(Constants.RES_CONFIG_DISPLAYER_UNSUPPORTED_DATATYPE));
               return;
@@ -272,6 +276,19 @@ private static void displayInputEnum(MEnumInput input) {
     print(input.getValue());
   }
 
+  /**
+   * Display content of List input
+   *
+   * @param input List input
+   */
+  private static void displayInputList(MListInput input) {
+    for (String element : input.getValue()) {
+      println();
+      print("      ");
+      print(element);
+    }
+  }
+
   private ConfigDisplayer() {
     // Do not instantiate
   }

shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java

@@ -29,6 +29,7 @@
 import org.apache.sqoop.model.MConfig;
 import org.apache.sqoop.model.MInput;
 import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MListInput;
 import org.apache.sqoop.model.MLongInput;
 import org.apache.sqoop.model.MMapInput;
 import org.apache.sqoop.model.MJob;
@@ -39,6 +40,7 @@
 import org.apache.sqoop.validation.Status;
 
 import java.io.IOException;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.HashMap;
@@ -197,12 +199,44 @@ public static boolean fillInput(String prefix, MInput input, CommandLine line) t
     case ENUM:
       assert input instanceof MEnumInput;
       return fillInputEnum(prefix, (MEnumInput) input, line);
+    case LIST:
+      assert input instanceof MListInput;
+      return fillInputList(prefix, (MListInput) input, line);
     default:
       println("Unsupported data type " + input.getType());
       return true;
     }
   }
 
+  /**
+   * Load CLI options for list type.
+   *
+   * Parses elements that take the config "<element1>&<element2>&...".
+   *
+   * @param prefix placed at the beginning of the CLI option key
+   * @param input Input that we should read or edit
+   * @param line CLI options container
+   * @return
+   * @throws IOException
+   */
+  private static boolean fillInputList(String prefix,
+                                       MListInput input,
+                                       CommandLine line)
+                                       throws IOException {
+    String opt = ConfigOptions.getOptionKey(prefix, input);
+    if (line.hasOption(opt)) {
+      String value = line.getOptionValue(opt);
+      List<String> values = new LinkedList<String>();
+      for (String element : value.split("&")) {
+        values.add(element);
+      }
+      input.setValue(values);
+    } else {
+      input.setEmpty();
+    }
+    return true;
+  }
+
   /**
    * Load CLI option for enum type.
    *
@@ -503,12 +537,80 @@ static boolean fillInputWithBundle(MInput input, ConsoleReader reader, ResourceB
         return fillInputMapWithBundle((MMapInput) input, reader, bundle);
       case ENUM:
         return fillInputEnumWithBundle((MEnumInput) input, reader, bundle);
+      case LIST:
+        return fillInputListWithBundle((MListInput) input, reader, bundle);
       default:
         println("Unsupported data type " + input.getType());
         return true;
     }
   }
 
+  /**
+   * Load user input for list type.
+   *
+   * This implementation will load one element at a time. If user did not
+   * enter anything (empty input) finish loading and return from function.
+   *
+   * @param input Input that we should read or edit
+   * @param reader Associated console reader
+   * @param bundle Resource bundle
+   * @return True if user wish to continue with loading additional inputs
+   * @throws IOException
+   */
+  private static boolean fillInputListWithBundle(MListInput input,
+                                       ConsoleReader reader,
+                                       ResourceBundle bundle)
+                                       throws IOException {
+    // Special prompt in List case
+    println(bundle.getString(input.getLabelKey()) + ": ");
+
+    // Internal loading list
+    List<String> values = input.getValue();
+    if(values == null) {
+      values = new LinkedList<String>();
+    }
+
+    String userTyped;
+
+    while(true) {
+      // Print all current items in each iteration
+      // However do not printout if this input contains sensitive information.
+      println("There are currently " + values.size() + " values in the list:");
+      if (!input.isSensitive()) {
+        for(String value : values) {
+          println(value);
+        }
+      }
+
+      // Special prompt for List element
+      reader.printString("element# ");
+      reader.flushConsole();
+
+      if(input.isSensitive()) {
+        userTyped = reader.readLine('*');
+      } else {
+        userTyped = reader.readLine();
+      }
+
+      if(userTyped == null) {
+        // Finish loading and return back to Sqoop shell
+        return false;
+      } else if(userTyped.isEmpty()) {
+        // User has finished loading data to List input, either set input empty
+        // if there are no elements or propagate elements to the input
+        if(values.size() == 0) {
+          input.setEmpty();
+        } else {
+          input.setValue(values);
+        }
+        return true;
+      } else {
+        values.add(userTyped);
+      }
+
+    }
+  }
+
   /**
    * Load user input for enum type.
    *