diff --git a/src/java/com/cloudera/sqoop/lib/SqoopRecord.java b/src/java/com/cloudera/sqoop/lib/SqoopRecord.java index 90d075aa..507b3f1f 100644 --- a/src/java/com/cloudera/sqoop/lib/SqoopRecord.java +++ b/src/java/com/cloudera/sqoop/lib/SqoopRecord.java @@ -96,5 +96,17 @@ public Map getFieldMap() { throw new RuntimeException( "Got null field map from record. Regenerate your record class."); } + + /** + * Allows an arbitrary field to be set programmatically to the + * specified value object. The value object must match the + * type expected for the particular field or a RuntimeException + * will result. + * @throws RuntimeException if the specified field name does not exist. + */ + public void setField(String fieldName, Object fieldVal) { + throw new RuntimeException("This SqoopRecord does not support setField(). " + + "Regenerate your record class."); + } } diff --git a/src/java/com/cloudera/sqoop/orm/ClassWriter.java b/src/java/com/cloudera/sqoop/orm/ClassWriter.java index 9e3a454e..ef0007c3 100644 --- a/src/java/com/cloudera/sqoop/orm/ClassWriter.java +++ b/src/java/com/cloudera/sqoop/orm/ClassWriter.java @@ -649,6 +649,41 @@ private void generateCloneMethod(Map columnTypes, sb.append(" }\n\n"); } + /** + * Generate the setField() method. + * @param columnTypes - mapping from column names to sql types + * @param colNames - ordered list of column names for table. + * @param sb - StringBuilder to append code to + */ + private void generateSetField(Map columnTypes, + String [] colNames, StringBuilder sb) { + sb.append(" public void setField(String __fieldName, Object __fieldVal) " + + "{\n"); + boolean first = true; + for (String colName : colNames) { + int sqlType = columnTypes.get(colName); + String javaType = connManager.toJavaType(sqlType); + if (null == javaType) { + continue; + } else { + if (!first) { + sb.append(" else"); + } + + sb.append(" if (\"" + colName + "\".equals(__fieldName)) {\n"); + sb.append(" this." + colName + " = (" + javaType + + ") __fieldVal;\n"); + sb.append(" }\n"); + first = false; + } + } + sb.append(" else {\n"); + sb.append(" throw new RuntimeException("); + sb.append("\"No such field: \" + __fieldName);\n"); + sb.append(" }\n"); + sb.append(" }\n"); + } + /** * Generate the getFieldMap() method. * @param columnTypes - mapping from column names to sql types @@ -1131,6 +1166,7 @@ private StringBuilder generateClassForColumns( generateParser(columnTypes, colNames, sb); generateCloneMethod(columnTypes, colNames, sb); generateGetFieldMap(columnTypes, colNames, sb); + generateSetField(columnTypes, colNames, sb); // TODO(aaron): Generate hashCode(), compareTo(), equals() so it can be a // WritableComparable diff --git a/src/test/com/cloudera/sqoop/orm/TestParseMethods.java b/src/test/com/cloudera/sqoop/orm/TestParseMethods.java index 3e598b73..3dbaab64 100644 --- a/src/test/com/cloudera/sqoop/orm/TestParseMethods.java +++ b/src/test/com/cloudera/sqoop/orm/TestParseMethods.java @@ -34,6 +34,8 @@ import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException; import com.cloudera.sqoop.config.ConfigurationHelper; + +import com.cloudera.sqoop.testutil.ExplicitSetMapper; import com.cloudera.sqoop.tool.ImportTool; import com.cloudera.sqoop.util.ClassLoaderStack; @@ -205,5 +207,66 @@ public void testNumericTypes() throws IOException { createTableWithColTypes(types, vals); runParseTest(",", "\\n", "\\\'", "\\", false); } + + public void testFieldSetter() throws IOException { + ClassLoader prevClassLoader = null; + + String [] types = { "VARCHAR(32)", "VARCHAR(32)" }; + String [] vals = { "'meep'", "'foo'" }; + createTableWithColTypes(types, vals); + + String [] argv = getArgv(true, ",", "\\n", "\\\'", "\\", false); + runImport(argv); + try { + String tableClassName = getTableName(); + + argv = getArgv(false, ",", "\\n", "\\\'", "\\", false); + SqoopOptions opts = new ImportTool().parseArguments(argv, null, null, + true); + + CompilationManager compileMgr = new CompilationManager(opts); + String jarFileName = compileMgr.getJarFilename(); + + // Make sure the user's class is loaded into our address space. + prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, + tableClassName); + + JobConf job = new JobConf(); + job.setJar(jarFileName); + + // Tell the job what class we're testing. + job.set(ExplicitSetMapper.USER_TYPE_NAME_KEY, tableClassName); + job.set(ExplicitSetMapper.SET_COL_KEY, BASE_COL_NAME + "0"); + job.set(ExplicitSetMapper.SET_VAL_KEY, "this-is-a-test"); + + // use local mode in the same JVM. + ConfigurationHelper.setJobtrackerAddr(job, "local"); + if (!BaseSqoopTestCase.isOnPhysicalCluster()) { + job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); + } + String warehouseDir = getWarehouseDir(); + Path warehousePath = new Path(warehouseDir); + Path inputPath = new Path(warehousePath, getTableName()); + Path outputPath = new Path(warehousePath, getTableName() + "-out"); + + job.setMapperClass(ExplicitSetMapper.class); + job.setNumReduceTasks(0); + FileInputFormat.addInputPath(job, inputPath); + FileOutputFormat.setOutputPath(job, outputPath); + + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(NullWritable.class); + + JobClient.runJob(job); + } catch (InvalidOptionsException ioe) { + fail(ioe.toString()); + } catch (ParseException pe) { + fail(pe.toString()); + } finally { + if (null != prevClassLoader) { + ClassLoaderStack.setCurrentClassLoader(prevClassLoader); + } + } + } } diff --git a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java index 4376ef40..3b1c76af 100644 --- a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java +++ b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java @@ -251,7 +251,7 @@ public void tearDown() { guaranteeCleanWarehouse(); } - static final String BASE_COL_NAME = "DATA_COL"; + public static final String BASE_COL_NAME = "DATA_COL"; protected String getColName(int i) { return BASE_COL_NAME + i; diff --git a/src/test/com/cloudera/sqoop/testutil/ExplicitSetMapper.java b/src/test/com/cloudera/sqoop/testutil/ExplicitSetMapper.java new file mode 100644 index 00000000..22ec794a --- /dev/null +++ b/src/test/com/cloudera/sqoop/testutil/ExplicitSetMapper.java @@ -0,0 +1,109 @@ +/** + * Licensed to Cloudera, Inc. under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Cloudera, Inc. licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.cloudera.sqoop.testutil; + +import java.io.IOException; + +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MapReduceBase; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; +import com.cloudera.sqoop.lib.SqoopRecord; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * Test harness mapper. Instantiate the user's specific type, explicitly + * set the value of a field with setField(), and read the field value + * back via the field map. Throw an IOException if it doesn't get set + * correctly. + */ +public class ExplicitSetMapper extends MapReduceBase + implements Mapper { + + public static final Log LOG = LogFactory.getLog( + ExplicitSetMapper.class.getName()); + + public static final String USER_TYPE_NAME_KEY = "sqoop.user.class"; + public static final String SET_COL_KEY = "sqoop.explicit.set.col"; + public static final String SET_VAL_KEY = "sqoop.explicit.set.val"; + + private SqoopRecord userRecord; + private String setCol; + private String setVal; + + public void configure(JobConf job) { + String userTypeName = job.get(USER_TYPE_NAME_KEY); + if (null == userTypeName) { + throw new RuntimeException("Unconfigured parameter: " + + USER_TYPE_NAME_KEY); + } + + setCol = job.get(SET_COL_KEY); + setVal = job.get(SET_VAL_KEY); + + LOG.info("User type name set to " + userTypeName); + LOG.info("Will try to set col " + setCol + " to " + setVal); + + this.userRecord = null; + + try { + Configuration conf = new Configuration(); + Class userClass = Class.forName(userTypeName, true, + Thread.currentThread().getContextClassLoader()); + this.userRecord = + (SqoopRecord) ReflectionUtils.newInstance(userClass, conf); + } catch (ClassNotFoundException cnfe) { + // handled by the next block. + LOG.error("ClassNotFound exception: " + cnfe.toString()); + } catch (Exception e) { + LOG.error("Got an exception reflecting user class: " + e.toString()); + } + + if (null == this.userRecord) { + LOG.error("Could not instantiate user record of type " + userTypeName); + throw new RuntimeException("Could not instantiate user record of type " + + userTypeName); + } + } + + public void map(LongWritable key, Text val, + OutputCollector out, Reporter r) throws IOException { + + // Try to set the field. + userRecord.setField(setCol, setVal); + Map fieldVals = userRecord.getFieldMap(); + if (!fieldVals.get(setCol).equals(setVal)) { + throw new IOException("Could not set column value! Got back " + + fieldVals.get(setCol)); + } else { + LOG.info("Correctly changed value for col " + setCol + " to " + setVal); + } + } +} +