From fa815557c58cbc22352d0b2aafeaf8e0ce71f034 Mon Sep 17 00:00:00 2001 From: Bilung Lee Date: Fri, 9 Nov 2012 18:27:09 -0800 Subject: [PATCH] SQOOP-646 Provide support for "set" metadata in Sqoop 2 (Jarek Jarcec Cecho) --- .../sqoop/client/utils/FormDisplayer.java | 5 + .../apache/sqoop/client/utils/FormFiller.java | 76 +++++++++- .../sqoop/json/util/FormSerialization.java | 14 ++ .../org/apache/sqoop/model/FormUtils.java | 16 +- .../org/apache/sqoop/model/MEnumInput.java | 123 +++++++++++++++ .../java/org/apache/sqoop/model/MInput.java | 8 +- .../org/apache/sqoop/model/MInputType.java | 3 + .../org/apache/sqoop/model/MIntegerInput.java | 10 -- .../org/apache/sqoop/model/MMapInput.java | 10 -- .../org/apache/sqoop/model/MStringInput.java | 4 +- .../org/apache/sqoop/model/ModelError.java | 2 + .../org/apache/sqoop/utils/ClassUtils.java | 34 ++++- .../apache/sqoop/utils/ClassUtils.java.orig | 141 ++++++++++++++++++ .../org/apache/sqoop/utils/StringUtils.java | 51 +++++++ .../apache/sqoop/utils/TestClassUtils.java | 31 ++++ .../apache/sqoop/utils/TestStringUtils.java | 35 +++++ .../configuration/ExportJobConfiguration.java | 2 +- .../configuration/ImportJobConfiguration.java | 2 +- .../framework/configuration/StorageType.java | 28 ++++ .../resources/framework-resources.properties | 8 +- .../derby/DerbyRepositoryHandler.java | 16 +- .../derby/DerbySchemaConstants.java | 2 + .../repository/derby/DerbySchemaQuery.java | 31 ++-- 23 files changed, 603 insertions(+), 49 deletions(-) create mode 100644 common/src/main/java/org/apache/sqoop/model/MEnumInput.java create mode 100644 common/src/main/java/org/apache/sqoop/utils/ClassUtils.java.orig create mode 100644 common/src/main/java/org/apache/sqoop/utils/StringUtils.java create mode 100644 common/src/test/java/org/apache/sqoop/utils/TestStringUtils.java create mode 100644 core/src/main/java/org/apache/sqoop/framework/configuration/StorageType.java diff --git a/client/src/main/java/org/apache/sqoop/client/utils/FormDisplayer.java b/client/src/main/java/org/apache/sqoop/client/utils/FormDisplayer.java index 96abd42a..97eea795 100644 --- a/client/src/main/java/org/apache/sqoop/client/utils/FormDisplayer.java +++ b/client/src/main/java/org/apache/sqoop/client/utils/FormDisplayer.java @@ -17,12 +17,14 @@ */ package org.apache.sqoop.client.utils; +import org.apache.sqoop.model.MEnumInput; import org.apache.sqoop.model.MForm; import org.apache.sqoop.model.MFramework; import org.apache.sqoop.model.MInput; import org.apache.sqoop.model.MInputType; import org.apache.sqoop.model.MJobForms; import org.apache.sqoop.model.MStringInput; +import org.apache.sqoop.utils.StringUtils; import org.codehaus.groovy.tools.shell.IO; import java.util.Iterator; @@ -101,6 +103,9 @@ public static void displayFormsMetadata(IO io, io.out.println(((MStringInput)input).isMasked()); io.out.print(" Size: "); io.out.println(((MStringInput)input).getMaxLength()); + } else if(input.getType() == MInputType.ENUM) { + io.out.print(" Possible values: "); + io.out.println(StringUtils.join(((MEnumInput)input).getValues(), ",")); } } } diff --git a/client/src/main/java/org/apache/sqoop/client/utils/FormFiller.java b/client/src/main/java/org/apache/sqoop/client/utils/FormFiller.java index 55d1cc00..753d78e7 100644 --- a/client/src/main/java/org/apache/sqoop/client/utils/FormFiller.java +++ b/client/src/main/java/org/apache/sqoop/client/utils/FormFiller.java @@ -20,6 +20,7 @@ import jline.ConsoleReader; import org.apache.sqoop.client.core.Environment; import org.apache.sqoop.model.MConnection; +import org.apache.sqoop.model.MEnumInput; import org.apache.sqoop.model.MForm; import org.apache.sqoop.model.MInput; import org.apache.sqoop.model.MIntegerInput; @@ -180,13 +181,86 @@ public static boolean fillInput(IO io, case INTEGER: return fillInputInteger(io, (MIntegerInput) input, reader, bundle); case MAP: - return fillInputMap(io, (MMapInput)input, reader, bundle); + return fillInputMap(io, (MMapInput) input, reader, bundle); + case ENUM: + return fillInputEnum(io, (MEnumInput) input, reader, bundle); default: io.out.println("Unsupported data type " + input.getType()); return true; } } + /** + * Load user input for enum type. + * + * Print out numbered list of all available options and let user choose one + * item from that. + * + * @param io Shell's IO object + * @param input Input that we should read or edit + * @param reader Associated console reader + * @param bundle Resource bundle + * @return True if user with to continue with loading addtional inputs + * @throws IOException + */ + private static boolean fillInputEnum(IO io, + MEnumInput input, + ConsoleReader reader, + ResourceBundle bundle) + throws IOException { + // Prompt in enum case + io.out.println(bundle.getString(input.getLabelKey()) + ": "); + + // Indexes + int i = -1; + int lastChoice = -1; + + // Print out all values as a numbered list + for(String value : input.getValues()) { + i++; + + io.out.println(" " + i + " : " + value); + + if(!input.isEmpty() && value.equals(input.getValue())) { + lastChoice = i; + } + } + + // Prompt + reader.printString("Choose: "); + + // Fill previously filled index when available + if(lastChoice != -1) { + reader.putString(Integer.toString(lastChoice)); + } + + reader.flushConsole(); + String userTyped = reader.readLine(); + + if (userTyped == null) { + return false; + } else if (userTyped.isEmpty()) { + input.setEmpty(); + } else { + Integer index; + try { + index = Integer.valueOf(userTyped); + + if(index < 0 || index >= input.getValues().length) { + errorMessage(io, "Invalid index"); + return fillInputEnum(io, input, reader, bundle); + } + + input.setValue(input.getValues()[index]); + } catch (NumberFormatException ex) { + errorMessage(io, "Input is not valid integer number"); + return fillInputEnum(io, input, reader, bundle); + } + } + + return true; + } + /** * Load user input for map type. * diff --git a/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java index 70fa8bf0..3d69bf86 100644 --- a/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java +++ b/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java @@ -17,6 +17,7 @@ */ package org.apache.sqoop.json.util; +import org.apache.sqoop.model.MEnumInput; import org.apache.sqoop.model.MForm; import org.apache.sqoop.model.MFormType; import org.apache.sqoop.model.MInput; @@ -24,6 +25,7 @@ import org.apache.sqoop.model.MIntegerInput; import org.apache.sqoop.model.MMapInput; import org.apache.sqoop.model.MStringInput; +import org.apache.sqoop.utils.StringUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -49,6 +51,7 @@ public final class FormSerialization { public static final String FORM_INPUT_MASK = "mask"; public static final String FORM_INPUT_SIZE = "size"; public static final String FORM_INPUT_VALUE = "value"; + public static final String FORM_INPUT_VALUES = "values"; /** * Transform given list of forms to JSON Array object. @@ -97,6 +100,12 @@ public static JSONObject extractForm(MForm mForm) { ((MStringInput)mInput).getMaxLength()); } + // Enum specific serialization + if(mInput.getType() == MInputType.ENUM) { + input.put(FORM_INPUT_VALUES, + StringUtils.join(((MEnumInput)mInput).getValues(), ",")); + } + // Serialize value if is there if(!mInput.isEmpty()) { input.put(FORM_INPUT_VALUE, mInput.getUrlSafeValueString()); @@ -153,6 +162,11 @@ public static MForm restoreForm(JSONObject form) { mInput = new MIntegerInput(name); break; } + case ENUM: { + String values = (String) input.get(FORM_INPUT_VALUES); + mInput = new MEnumInput(name, values.split(",")); + break; + } } // Propagate form ID diff --git a/common/src/main/java/org/apache/sqoop/model/FormUtils.java b/common/src/main/java/org/apache/sqoop/model/FormUtils.java index ee927189..418b0a7d 100644 --- a/common/src/main/java/org/apache/sqoop/model/FormUtils.java +++ b/common/src/main/java/org/apache/sqoop/model/FormUtils.java @@ -17,14 +17,13 @@ */ package org.apache.sqoop.model; -import org.apache.log4j.Logger; import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.utils.ClassUtils; import org.apache.sqoop.validation.Status; import org.apache.sqoop.validation.Validation; import org.json.simple.JSONObject; import org.json.simple.JSONValue; -import java.lang.annotation.Annotation; import java.lang.reflect.Field; import java.util.HashMap; import java.util.LinkedList; @@ -111,6 +110,8 @@ public static List toForms(Class klass, Object configuration) { input = new MMapInput(fieldName); } else if(type == Integer.class) { input = new MIntegerInput(fieldName); + } else if(type.isEnum()) { + input = new MEnumInput(fieldName, ClassUtils.getEnumStrings(type)); } else { throw new SqoopException(ModelError.MODEL_004, "Unsupported type " + type.getName() + " for input " + fieldName); @@ -167,11 +168,16 @@ public static void fillValues(List forms, Object configuration) { // We need to access this field even if it would be declared as private field.setAccessible(true); + // Propagate value to the configuration object try { if(input.isEmpty()) { field.set(configuration, null); } else { - field.set(configuration, input.getValue()); + if (input.getType() == MInputType.ENUM) { + field.set(configuration, Enum.valueOf((Class)field.getType(), (String) input.getValue())); + } else { + field.set(configuration, input.getValue()); + } } } catch (IllegalAccessException e) { throw new SqoopException(ModelError.MODEL_005, @@ -256,6 +262,8 @@ public static String toJson(Object configuration) { jsonObject.put(fieldName, map); } else if(type == Integer.class) { jsonObject.put(fieldName, value); + } else if(type.isEnum()) { + jsonObject.put(fieldName, value); } else { throw new SqoopException(ModelError.MODEL_004, "Unsupported type " + type.getName() + " for input " + fieldName); @@ -300,6 +308,8 @@ public static void fillValues(String json, Object configuration) { field.set(key, map); } else if(type == Integer.class) { field.set(configuration, jsonObject.get(key)); + } else if(type == Integer.class) { + field.set(configuration, Enum.valueOf((Class)field.getType(), (String) jsonObject.get(key))); } else { throw new SqoopException(ModelError.MODEL_004, "Unsupported type " + type.getName() + " for input " + key); diff --git a/common/src/main/java/org/apache/sqoop/model/MEnumInput.java b/common/src/main/java/org/apache/sqoop/model/MEnumInput.java new file mode 100644 index 00000000..bed2b040 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/model/MEnumInput.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.model; + +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.utils.StringUtils; + +import java.util.Arrays; + +/** + * + */ +public class MEnumInput extends MInput { + + /** + * Array of available values + */ + String []values; + + public MEnumInput(String name, String[] values) { + super(name); + this.values = values; + } + + public String[] getValues() { + return values; + } + + @Override + public void setValue(String value) { + // Null is allowed value + if(value == null) { + super.setValue(null); + return; + } + + // However non null values must be available from given enumeration list + for(String allowedValue : values) { + if(allowedValue.equals(value)) { + super.setValue(value); + return; + } + } + + // Otherwise we've got invalid value + throw new SqoopException(ModelError.MODEL_008, + "Invalid value " + value); + } + + @Override + public String getUrlSafeValueString() { + return getValue(); + } + + @Override + public void restoreFromUrlSafeValueString(String valueString) { + setValue(valueString); + } + + @Override + public MInputType getType() { + return MInputType.ENUM; + } + + @Override + public boolean hasExtraInfo() { + return true; + } + + @Override + public String getExtraInfoToString() { + return StringUtils.join(values, ","); + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (!(other instanceof MEnumInput)) { + return false; + } + + MEnumInput mei = (MEnumInput) other; + return getName().equals(mei.getName()) && Arrays.equals(values, mei.values); + } + + @Override + public int hashCode() { + int hash = 23 + 31 * getName().hashCode(); + for(String value : values) { + hash += 31 * value.hashCode(); + } + + return hash; + } + + @Override + public boolean isEmpty() { + return getValue() == null; + } + + @Override + public void setEmpty() { + setValue(null); + } +} diff --git a/common/src/main/java/org/apache/sqoop/model/MInput.java b/common/src/main/java/org/apache/sqoop/model/MInput.java index 2a87531e..96397e82 100644 --- a/common/src/main/java/org/apache/sqoop/model/MInput.java +++ b/common/src/main/java/org/apache/sqoop/model/MInput.java @@ -62,13 +62,17 @@ public T getValue() { * @return true if this type maintains more state than what is * stored in the MInput base class. */ - protected abstract boolean hasExtraInfo(); + public boolean hasExtraInfo() { + return false; + } /** * @return the string representation of state stored in this type if * applicable or an empty string. */ - protected abstract String getExtraInfoToString(); + public String getExtraInfoToString() { + return null; + } /** * All input types must override the equals() method such that the diff --git a/common/src/main/java/org/apache/sqoop/model/MInputType.java b/common/src/main/java/org/apache/sqoop/model/MInputType.java index 80163493..76411e33 100644 --- a/common/src/main/java/org/apache/sqoop/model/MInputType.java +++ b/common/src/main/java/org/apache/sqoop/model/MInputType.java @@ -34,5 +34,8 @@ public enum MInputType { /** Integer input type */ INTEGER, + /** String based input that can contain only predefined values **/ + ENUM, + ; } diff --git a/common/src/main/java/org/apache/sqoop/model/MIntegerInput.java b/common/src/main/java/org/apache/sqoop/model/MIntegerInput.java index 26477cc2..d281d7ef 100644 --- a/common/src/main/java/org/apache/sqoop/model/MIntegerInput.java +++ b/common/src/main/java/org/apache/sqoop/model/MIntegerInput.java @@ -51,16 +51,6 @@ public MInputType getType() { return MInputType.INTEGER; } - @Override - protected boolean hasExtraInfo() { - return false; - } - - @Override - protected String getExtraInfoToString() { - return ""; - } - @Override public boolean equals(Object other) { if (other == this) { diff --git a/common/src/main/java/org/apache/sqoop/model/MMapInput.java b/common/src/main/java/org/apache/sqoop/model/MMapInput.java index de44d3f6..b4580221 100644 --- a/common/src/main/java/org/apache/sqoop/model/MMapInput.java +++ b/common/src/main/java/org/apache/sqoop/model/MMapInput.java @@ -75,16 +75,6 @@ public MInputType getType() { return MInputType.MAP; } - @Override - protected boolean hasExtraInfo() { - return false; - } - - @Override - protected String getExtraInfoToString() { - return ""; - } - @Override public boolean equals(Object other) { if (other == this) { diff --git a/common/src/main/java/org/apache/sqoop/model/MStringInput.java b/common/src/main/java/org/apache/sqoop/model/MStringInput.java index 04afb389..e96ec929 100644 --- a/common/src/main/java/org/apache/sqoop/model/MStringInput.java +++ b/common/src/main/java/org/apache/sqoop/model/MStringInput.java @@ -72,12 +72,12 @@ public MInputType getType() { } @Override - protected boolean hasExtraInfo() { + public boolean hasExtraInfo() { return true; } @Override - protected String getExtraInfoToString() { + public String getExtraInfoToString() { return isMasked() + ":" + getMaxLength(); } diff --git a/common/src/main/java/org/apache/sqoop/model/ModelError.java b/common/src/main/java/org/apache/sqoop/model/ModelError.java index d3467c06..2c496cec 100644 --- a/common/src/main/java/org/apache/sqoop/model/ModelError.java +++ b/common/src/main/java/org/apache/sqoop/model/ModelError.java @@ -38,6 +38,8 @@ public enum ModelError implements ErrorCode { MODEL_007("Primitive types in configuration objects are not allowed"), + MODEL_008("Invalid input value"), + ; private final String message; diff --git a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java index b63fce2a..4f89b505 100644 --- a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java +++ b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java @@ -21,6 +21,10 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; public final class ClassUtils { @@ -121,10 +125,9 @@ public static Object instantiate(Class klass, Object ... args) { */ public static String jarForClass(String className) { Class klass = loadClass(className); - return klass.getProtectionDomain().getCodeSource().getLocation().toString(); + return jarForClass(klass); } - /** * Return jar path for given class. * @@ -135,6 +138,33 @@ public static String jarForClass(Class klass) { return klass.getProtectionDomain().getCodeSource().getLocation().toString(); } + /** + * Get list of constants from given Enum as a array of strings. + * + * @param klass Enumeration class + * @return Array of string representation or null in case of any error + */ + public static String[] getEnumStrings(Class klass) { + if(!klass.isEnum()) { + return null; + } + + ArrayList values = new ArrayList(); + + try { + Method methodName = klass.getMethod("name"); + + for(Object constant : klass.getEnumConstants()) { + values.add((String) methodName.invoke(constant)); + } + } catch (Exception e) { + LOG.error("Can't get list of values from enumeration " + klass.getCanonicalName(), e); + return null; + } + + return values.toArray(new String[values.size()]); + } + private ClassUtils() { // Disable explicit object creation } diff --git a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java.orig b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java.orig new file mode 100644 index 00000000..b63fce2a --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java.orig @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.utils; + +import org.apache.log4j.Logger; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +public final class ClassUtils { + + private static final Logger LOG = Logger.getLogger(ClassUtils.class); + + /** + * Load class by given name and return corresponding Class object. + * + * This method will return null in case that the class is not found, no + * exception will be rised. + * + * @param className Name of class + * @return Class instance or NULL + */ + public static Class loadClass(String className) { + if(className == null) { + return null; + } + + Class klass = null; + try { + klass = Class.forName(className); + } catch (ClassNotFoundException ex) { + LOG.debug("Exception while loading class: " + className, ex); + } + + if (klass == null) { + // Try the context class loader if one exists + ClassLoader ctxLoader = Thread.currentThread().getContextClassLoader(); + if (ctxLoader != null) { + try { + klass = ctxLoader.loadClass(className); + } catch (ClassNotFoundException ex) { + LOG.debug("Exception while load class: " + className, ex); + } + } + } + + return klass; + } + + /** + * Create instance of given class and given parameters. + * + * Please note that due to inherited limitations from Java languge, this + * method can't handle primitive types and NULL values. + * + * @param className Class name + * @param args Objects that should be passed as constructor arguments. + * @return Instance of new class or NULL in case of any error + */ + public static Object instantiate(String className, Object ... args) { + return instantiate(loadClass(className), args); + } + + /** + * Create instance of given class and given parameters. + * + * Please note that due to inherited limitations from Java languge, this + * method can't handle primitive types and NULL values. + * + * @param klass Class object + * @param args Objects that should be passed as constructor arguments. + * @return Instance of new class or NULL in case of any error + */ + public static Object instantiate(Class klass, Object ... args) { + if(klass == null) { + return null; + } + + Class []argumentTypes = new Class[args.length]; + for(int i = 0; i < args.length; i++) { + Class type = args[i].getClass(); + argumentTypes[i] = type; + } + + try { + Constructor constructor = klass.getConstructor(argumentTypes); + return constructor.newInstance(args); + } catch (NoSuchMethodException e) { + LOG.error("Can't find such constructor.", e); + } catch (InvocationTargetException e) { + LOG.error("Can't instantiate object.", e); + } catch (InstantiationException e) { + LOG.error("Can't instantiate object.", e); + } catch (IllegalAccessException e) { + LOG.error("Can't instantiate object.", e); + } + + return null; + } + + /** + * Return jar path for given class. + * + * @param className Class name + * @return Path on local filesystem to jar where given jar is present + */ + public static String jarForClass(String className) { + Class klass = loadClass(className); + return klass.getProtectionDomain().getCodeSource().getLocation().toString(); + } + + + /** + * Return jar path for given class. + * + * @param klass Class object + * @return Path on local filesystem to jar where given jar is present + */ + public static String jarForClass(Class klass) { + return klass.getProtectionDomain().getCodeSource().getLocation().toString(); + } + + private ClassUtils() { + // Disable explicit object creation + } +} diff --git a/common/src/main/java/org/apache/sqoop/utils/StringUtils.java b/common/src/main/java/org/apache/sqoop/utils/StringUtils.java new file mode 100644 index 00000000..97d356a4 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/utils/StringUtils.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.utils; + +/** + * Convenience String methods. + * + * We might consider replacing this with commons-lang library at some point. + */ +public final class StringUtils { + + public static String join(String []array, String separator) { + if(array == null) { + return null; + } + + StringBuilder sb = new StringBuilder(); + boolean first = true; + + for(String item : array) { + if(first) { + first = false; + } else { + sb.append(separator); + } + + sb.append(item); + } + + return sb.toString(); + } + + private StringUtils() { + // Instantiation is prohibited + } +} diff --git a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java index f0b0742d..a5c4e69e 100644 --- a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java +++ b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java @@ -65,4 +65,35 @@ public A(String a, Integer b, Integer c) { this.c = c; } } + + public void testGetEnumStrings() { + assertNull(ClassUtils.getEnumStrings(A.class)); + + assertEquals( + new String[] {"A", "B", "C"}, + ClassUtils.getEnumStrings(EnumA.class) + ); + assertEquals( + new String[] {"X", "Y"}, + ClassUtils.getEnumStrings(EnumX.class) + ); + } + + enum EnumX { + X, Y + } + + enum EnumA { + A, B, C + } + + public void assertEquals(String[] expected, String[] actual) { + assertEquals("Arrays do not have same length", expected.length, actual.length); + + for(int i = 0; i < expected.length; i++) { + assertEquals("Items on position " + i + " differs, expected " + + expected[i] + ", actual " + actual[i], + expected[i], actual[i]); + } + } } diff --git a/common/src/test/java/org/apache/sqoop/utils/TestStringUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestStringUtils.java new file mode 100644 index 00000000..e0ca2ec4 --- /dev/null +++ b/common/src/test/java/org/apache/sqoop/utils/TestStringUtils.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.utils; + +import junit.framework.TestCase; + +/** + * + */ +public class TestStringUtils extends TestCase { + + public void testJoin() { + assertNull(StringUtils.join(null, "a")); + + String []a = new String[] {"a", "b", "c"}; + + assertEquals("a,b,c", StringUtils.join(a, ",")); + assertEquals("abc", StringUtils.join(a, "")); + } +} diff --git a/core/src/main/java/org/apache/sqoop/framework/configuration/ExportJobConfiguration.java b/core/src/main/java/org/apache/sqoop/framework/configuration/ExportJobConfiguration.java index 9da51335..90541e96 100644 --- a/core/src/main/java/org/apache/sqoop/framework/configuration/ExportJobConfiguration.java +++ b/core/src/main/java/org/apache/sqoop/framework/configuration/ExportJobConfiguration.java @@ -29,5 +29,5 @@ public class ExportJobConfiguration { @Input(form = FORM_OUTPUT, size = 25) - public String outputFormat; + public String ignored; } diff --git a/core/src/main/java/org/apache/sqoop/framework/configuration/ImportJobConfiguration.java b/core/src/main/java/org/apache/sqoop/framework/configuration/ImportJobConfiguration.java index de8ddbca..d811593d 100644 --- a/core/src/main/java/org/apache/sqoop/framework/configuration/ImportJobConfiguration.java +++ b/core/src/main/java/org/apache/sqoop/framework/configuration/ImportJobConfiguration.java @@ -29,7 +29,7 @@ public class ImportJobConfiguration { @Input(form = FORM_OUTPUT, size = 25) - public String outputFormat; + public StorageType storageType; @Input(form = FORM_OUTPUT, size = 25) public String outputDirectory; diff --git a/core/src/main/java/org/apache/sqoop/framework/configuration/StorageType.java b/core/src/main/java/org/apache/sqoop/framework/configuration/StorageType.java new file mode 100644 index 00000000..dbe9f952 --- /dev/null +++ b/core/src/main/java/org/apache/sqoop/framework/configuration/StorageType.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.framework.configuration; + +/** + * Various storage types that Sqoop is supporting + */ +public enum StorageType { + /** + * Direct HDFS import + */ + HDFS, +} diff --git a/core/src/main/resources/framework-resources.properties b/core/src/main/resources/framework-resources.properties index 9f194690..872f48e9 100644 --- a/core/src/main/resources/framework-resources.properties +++ b/core/src/main/resources/framework-resources.properties @@ -32,8 +32,12 @@ form-output-label = Output configuration form-output-help = You must supply the information requested in order to \ get information where you want to store your data. -outputFormat-label = Output format -outputFormat-help = Output format that should be used +storageType-label = Storage type +storageType-help = Target on Hadoop ecosystem where to store data outputDirectory-label = Output directory outputDirectory-help = Output directory for final data + +ignored-label = Ignored +ignored-help = This value is ignored + diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java index 70062735..90aef4ba 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java @@ -40,6 +40,7 @@ import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.model.MConnection; import org.apache.sqoop.model.MConnectionForms; +import org.apache.sqoop.model.MEnumInput; import org.apache.sqoop.model.MIntegerInput; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MJobForms; @@ -56,6 +57,7 @@ import org.apache.sqoop.repository.JdbcRepositoryHandler; import org.apache.sqoop.repository.JdbcRepositoryTransactionFactory; import org.apache.sqoop.submission.SubmissionStatus; +import org.apache.sqoop.utils.StringUtils; /** * JDBC based repository handler for Derby database. @@ -1149,6 +1151,7 @@ private void registerFormInputs(long formId, List> inputs, baseInputStmt.setLong(2, formId); baseInputStmt.setShort(3, inputIndex++); baseInputStmt.setString(4, input.getType().name()); + // String specific column(s) if (input.getType().equals(MInputType.STRING)) { MStringInput strInput = (MStringInput) input; baseInputStmt.setBoolean(5, strInput.isMasked()); @@ -1157,6 +1160,13 @@ private void registerFormInputs(long formId, List> inputs, baseInputStmt.setNull(5, Types.BOOLEAN); baseInputStmt.setNull(6, Types.INTEGER); } + // Enum specific column(s) + if(input.getType() == MInputType.ENUM) { + baseInputStmt.setString(7, StringUtils.join(((MEnumInput)input).getValues(), ",")); + } else { + baseInputStmt.setNull(7, Types.VARCHAR); + } + int baseInputCount = baseInputStmt.executeUpdate(); if (baseInputCount != 1) { throw new SqoopException(DerbyRepoError.DERBYREPO_0017, @@ -1268,7 +1278,8 @@ public void loadForms(List connectionForms, String inputType = rsetInput.getString(5); boolean inputStrMask = rsetInput.getBoolean(6); short inputStrLength = rsetInput.getShort(7); - String value = rsetInput.getString(8); + String inputEnumValues = rsetInput.getString(8); + String value = rsetInput.getString(9); MInputType mit = MInputType.valueOf(inputType); @@ -1283,6 +1294,9 @@ public void loadForms(List connectionForms, case INTEGER: input = new MIntegerInput(inputName); break; + case ENUM: + input = new MEnumInput(inputName, inputEnumValues.split(",")); + break; default: throw new SqoopException(DerbyRepoError.DERBYREPO_0006, "input-" + inputName + ":" + inputId + ":" diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java index 1f106746..3664df6e 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java @@ -76,6 +76,8 @@ public final class DerbySchemaConstants { public static final String COLUMN_SQI_STRLENGTH = "SQI_STRLENGTH"; + public static final String COLUMN_SQI_ENUMVALS = "SQI_ENUMVALS"; + // SQ_CONNECTION public static final String TABLE_SQ_CONNECTION_NAME = "SQ_CONNECTION"; diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java index 93054457..496e3d9e 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java @@ -63,6 +63,7 @@ * | SQI_TYPE: VARCHAR(32) | "STRING"|"MAP" * | SQI_STRMASK: BOOLEAN | * | SQI_STRLENGTH: SMALLINT | + * | SQI_ENUMVALS: VARCHAR(100) | * +----------------------------+ * *

@@ -165,8 +166,9 @@ public final class DerbySchemaQuery { + "PRIMARY KEY, " + COLUMN_SQI_NAME + " VARCHAR(64), " + COLUMN_SQI_FORM + " BIGINT, " + COLUMN_SQI_INDEX + " SMALLINT, " + COLUMN_SQI_TYPE + " VARCHAR(32), " + COLUMN_SQI_STRMASK + " BOOLEAN, " - + COLUMN_SQI_STRLENGTH + " SMALLINT, FOREIGN KEY (" + COLUMN_SQI_FORM - + ") REFERENCES " + TABLE_SQ_FORM + " (" + COLUMN_SQF_ID + "))"; + + COLUMN_SQI_STRLENGTH + " SMALLINT, " + COLUMN_SQI_ENUMVALS + + " VARCHAR(100), FOREIGN KEY (" + COLUMN_SQI_FORM + ") REFERENCES " + + TABLE_SQ_FORM + " (" + COLUMN_SQF_ID + "))"; // DDL: Create table SQ_CONNECTION public static final String QUERY_CREATE_TABLE_SQ_CONNECTION = @@ -242,18 +244,19 @@ public final class DerbySchemaQuery { public static final String STMT_FETCH_INPUT = "SELECT " + COLUMN_SQI_ID + ", " + COLUMN_SQI_NAME + ", " + COLUMN_SQI_FORM + ", " + COLUMN_SQI_INDEX + ", " + COLUMN_SQI_TYPE - + ", " + COLUMN_SQI_STRMASK + ", " + COLUMN_SQI_STRLENGTH - + ", cast(null as varchar(100)) FROM " + TABLE_SQ_INPUT + " WHERE " - + COLUMN_SQI_FORM + " = ? ORDER BY " + COLUMN_SQI_INDEX; + + ", " + COLUMN_SQI_STRMASK + ", " + COLUMN_SQI_STRLENGTH + ", " + + COLUMN_SQI_ENUMVALS + ", cast(null as varchar(100)) FROM " + + TABLE_SQ_INPUT + " WHERE " + COLUMN_SQI_FORM + " = ? ORDER BY " + + COLUMN_SQI_INDEX; // DML: Fetch inputs and values for a given connection public static final String STMT_FETCH_CONNECTION_INPUT = "SELECT " + COLUMN_SQI_ID + ", " + COLUMN_SQI_NAME + ", " + COLUMN_SQI_FORM + ", " + COLUMN_SQI_INDEX + ", " + COLUMN_SQI_TYPE - + ", " + COLUMN_SQI_STRMASK + ", " + COLUMN_SQI_STRLENGTH - + ", " + COLUMN_SQNI_VALUE + " FROM " + TABLE_SQ_INPUT - + " LEFT OUTER JOIN " + TABLE_SQ_CONNECTION_INPUT + " ON " - + COLUMN_SQNI_INPUT + " = " + COLUMN_SQI_ID + " AND " + + ", " + COLUMN_SQI_STRMASK + ", " + COLUMN_SQI_STRLENGTH + "," + + COLUMN_SQI_ENUMVALS + ", " + COLUMN_SQNI_VALUE + " FROM " + + TABLE_SQ_INPUT + " LEFT OUTER JOIN " + TABLE_SQ_CONNECTION_INPUT + + " ON " + COLUMN_SQNI_INPUT + " = " + COLUMN_SQI_ID + " AND " + COLUMN_SQNI_CONNECTION + " = ? WHERE " + COLUMN_SQI_FORM + " = ? AND (" + COLUMN_SQNI_CONNECTION + " = ? OR " + COLUMN_SQNI_CONNECTION + " IS NULL) ORDER BY " + COLUMN_SQI_INDEX; @@ -262,9 +265,9 @@ public final class DerbySchemaQuery { public static final String STMT_FETCH_JOB_INPUT = "SELECT " + COLUMN_SQI_ID + ", " + COLUMN_SQI_NAME + ", " + COLUMN_SQI_FORM + ", " + COLUMN_SQI_INDEX + ", " + COLUMN_SQI_TYPE - + ", " + COLUMN_SQI_STRMASK + ", " + COLUMN_SQI_STRLENGTH - + ", " + COLUMN_SQBI_VALUE + " FROM " + TABLE_SQ_INPUT - + " LEFT OUTER JOIN " + TABLE_SQ_JOB_INPUT + " ON " + + ", " + COLUMN_SQI_STRMASK + ", " + COLUMN_SQI_STRLENGTH + ", " + + COLUMN_SQI_ENUMVALS + ", " + COLUMN_SQBI_VALUE + " FROM " + + TABLE_SQ_INPUT + " LEFT OUTER JOIN " + TABLE_SQ_JOB_INPUT + " ON " + COLUMN_SQBI_INPUT + " = " + COLUMN_SQI_ID + " AND " + COLUMN_SQBI_JOB + " = ? WHERE " + COLUMN_SQI_FORM + " = ? AND (" + COLUMN_SQBI_JOB + " = ? OR " + COLUMN_SQBI_JOB + " IS NULL) ORDER BY " @@ -285,8 +288,8 @@ public final class DerbySchemaQuery { public static final String STMT_INSERT_INPUT_BASE = "INSERT INTO " + TABLE_SQ_INPUT + " (" + COLUMN_SQI_NAME + ", " + COLUMN_SQI_FORM + ", " + COLUMN_SQI_INDEX + ", " + COLUMN_SQI_TYPE - + ", " + COLUMN_SQI_STRMASK + ", " + COLUMN_SQI_STRLENGTH + ") " - + "VALUES (?, ?, ?, ?, ?, ?)"; + + ", " + COLUMN_SQI_STRMASK + ", " + COLUMN_SQI_STRLENGTH + ", " + + COLUMN_SQI_ENUMVALS + ") " + "VALUES (?, ?, ?, ?, ?, ?, ?)"; // DML: Insert new connection public static final String STMT_INSERT_CONNECTION =