From d2c40366a7458b0f4804a382403bb40ac72fbb4f Mon Sep 17 00:00:00 2001 From: Boglarka Egyed Date: Thu, 29 Mar 2018 16:09:31 +0200 Subject: [PATCH] SQOOP-3308: Mock ConnManager field in TestTableDefWriter (Szabolcs Vasas via Boglarka Egyed) --- .../org/apache/sqoop/hive/TableDefWriter.java | 33 +---- .../apache/sqoop/hive/TestTableDefWriter.java | 133 +++++++----------- 2 files changed, 54 insertions(+), 112 deletions(-) diff --git a/src/java/org/apache/sqoop/hive/TableDefWriter.java b/src/java/org/apache/sqoop/hive/TableDefWriter.java index e1424c38..b7a25b78 100644 --- a/src/java/org/apache/sqoop/hive/TableDefWriter.java +++ b/src/java/org/apache/sqoop/hive/TableDefWriter.java @@ -20,7 +20,6 @@ import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.Map; import java.util.Date; import java.text.DateFormat; @@ -79,17 +78,6 @@ public TableDefWriter(final SqoopOptions opts, final ConnManager connMgr, this.commentsEnabled = withComments; } - private Map externalColTypes; - - /** - * Set the column type map to be used. - * (dependency injection for testing; not used in production.) - */ - public void setColumnTypes(Map colTypes) { - this.externalColTypes = colTypes; - LOG.debug("Using test-controlled type map"); - } - /** * Get the column names to import. */ @@ -97,14 +85,6 @@ public void setColumnTypes(Map colTypes) { String [] colNames = options.getColumns(); if (null != colNames) { return colNames; // user-specified column names. - } else if (null != externalColTypes) { - // Test-injection column mapping. Extract the col names from this. - ArrayList keyList = new ArrayList(); - for (String key : externalColTypes.keySet()) { - keyList.add(key); - } - - return keyList.toArray(new String[keyList.size()]); } else if (null != inputTableName) { return connManager.getColumnNames(inputTableName); } else { @@ -119,16 +99,11 @@ public String getCreateTableStmt() throws IOException { Map columnTypes; Properties userMapping = options.getMapColumnHive(); Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); - if (externalColTypes != null) { - // Use pre-defined column types. - columnTypes = externalColTypes; + // Get these from the database. + if (null != inputTableName) { + columnTypes = connManager.getColumnTypes(inputTableName); } else { - // Get these from the database. - if (null != inputTableName) { - columnTypes = connManager.getColumnTypes(inputTableName); - } else { - columnTypes = connManager.getColumnTypesForQuery(options.getSqlQuery()); - } + columnTypes = connManager.getColumnTypesForQuery(options.getSqlQuery()); } String [] colNames = getColumnNames(); diff --git a/src/test/org/apache/sqoop/hive/TestTableDefWriter.java b/src/test/org/apache/sqoop/hive/TestTableDefWriter.java index 496b5add..8bdc3beb 100644 --- a/src/test/org/apache/sqoop/hive/TestTableDefWriter.java +++ b/src/test/org/apache/sqoop/hive/TestTableDefWriter.java @@ -23,12 +23,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.manager.ConnManager; import org.apache.sqoop.util.SqlTypeMap; import org.apache.sqoop.SqoopOptions; -import org.apache.sqoop.tool.ImportTool; import org.apache.sqoop.testutil.HsqldbTestServer; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -39,6 +40,9 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** @@ -49,9 +53,34 @@ public class TestTableDefWriter { public static final Log LOG = LogFactory.getLog( TestTableDefWriter.class.getName()); + private ConnManager connManager; + + private Configuration conf; + + private SqoopOptions options; + + private TableDefWriter writer; + + private String inputTable; + + private String outputTable; + @Rule public ExpectedException thrown = ExpectedException.none(); + @Before + public void before() { + inputTable = HsqldbTestServer.getTableName(); + outputTable = "outputTable"; + connManager = mock(ConnManager.class); + conf = new Configuration(); + options = new SqoopOptions(); + when(connManager.getColumnTypes(anyString())).thenReturn(new SqlTypeMap()); + when(connManager.getColumnNames(anyString())).thenReturn(new String[]{}); + + writer = new TableDefWriter(options, connManager, inputTable, outputTable, conf, false); + } + // Test getHiveOctalCharCode and expect an IllegalArgumentException. private void expectExceptionInCharCode(int charCode) { thrown.expect(IllegalArgumentException.class); @@ -73,14 +102,6 @@ public void testHiveOctalCharCode() { @Test public void testDifferentTableNames() throws Exception { - Configuration conf = new Configuration(); - SqoopOptions options = new SqoopOptions(); - TableDefWriter writer = new TableDefWriter(options, null, - "inputTable", "outputTable", conf, false); - - Map colTypes = new SqlTypeMap(); - writer.setColumnTypes(colTypes); - String createTable = writer.getCreateTableStmt(); String loadData = writer.getLoadDataStmt(); @@ -91,24 +112,15 @@ public void testDifferentTableNames() throws Exception { assertTrue(createTable.indexOf( "CREATE TABLE IF NOT EXISTS `outputTable`") != -1); assertTrue(loadData.indexOf("INTO TABLE `outputTable`") != -1); - assertTrue(loadData.indexOf("/inputTable'") != -1); + assertTrue(loadData.indexOf("/" + inputTable + "'") != -1); } @Test public void testDifferentTargetDirs() throws Exception { String targetDir = "targetDir"; - String inputTable = "inputTable"; - String outputTable = "outputTable"; - Configuration conf = new Configuration(); - SqoopOptions options = new SqoopOptions(); // Specify a different target dir from input table name options.setTargetDir(targetDir); - TableDefWriter writer = new TableDefWriter(options, null, - inputTable, outputTable, conf, false); - - Map colTypes = new SqlTypeMap(); - writer.setColumnTypes(colTypes); String createTable = writer.getCreateTableStmt(); String loadData = writer.getLoadDataStmt(); @@ -125,18 +137,8 @@ public void testDifferentTargetDirs() throws Exception { @Test public void testPartitions() throws Exception { - String[] args = { - "--hive-partition-key", "ds", - "--hive-partition-value", "20110413", - }; - Configuration conf = new Configuration(); - SqoopOptions options = - new ImportTool().parseArguments(args, null, null, false); - TableDefWriter writer = new TableDefWriter(options, - null, "inputTable", "outputTable", conf, false); - - Map colTypes = new SqlTypeMap(); - writer.setColumnTypes(colTypes); + options.setHivePartitionKey("ds"); + options.setHivePartitionValue("20110413"); String createTable = writer.getCreateTableStmt(); String loadData = writer.getLoadDataStmt(); @@ -152,18 +154,8 @@ public void testPartitions() throws Exception { @Test public void testLzoSplitting() throws Exception { - String[] args = { - "--compress", - "--compression-codec", "lzop", - }; - Configuration conf = new Configuration(); - SqoopOptions options = - new ImportTool().parseArguments(args, null, null, false); - TableDefWriter writer = new TableDefWriter(options, - null, "inputTable", "outputTable", conf, false); - - Map colTypes = new SqlTypeMap(); - writer.setColumnTypes(colTypes); + options.setUseCompression(true); + options.setCompressionCodec("lzop"); String createTable = writer.getCreateTableStmt(); String loadData = writer.getLoadDataStmt(); @@ -181,19 +173,13 @@ public void testLzoSplitting() throws Exception { @Test public void testUserMappingNoDecimal() throws Exception { - String[] args = { - "--map-column-hive", "id=STRING,value=INTEGER", - }; - Configuration conf = new Configuration(); - SqoopOptions options = - new ImportTool().parseArguments(args, null, null, false); - TableDefWriter writer = new TableDefWriter(options, - null, HsqldbTestServer.getTableName(), "outputTable", conf, false); + options.setMapColumnHive("id=STRING,value=INTEGER"); Map colTypes = new SqlTypeMap(); colTypes.put("id", Types.INTEGER); colTypes.put("value", Types.VARCHAR); - writer.setColumnTypes(colTypes); + + setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes); String createTable = writer.getCreateTableStmt(); @@ -208,15 +194,7 @@ public void testUserMappingNoDecimal() throws Exception { @Test public void testUserMappingWithDecimal() throws Exception { - String[] args = { - "--map-column-hive", "id=STRING,value2=DECIMAL(13,5),value1=INTEGER," + - "value3=DECIMAL(4,5),value4=VARCHAR(255)", - }; - Configuration conf = new Configuration(); - SqoopOptions options = - new ImportTool().parseArguments(args, null, null, false); - TableDefWriter writer = new TableDefWriter(options, - null, HsqldbTestServer.getTableName(), "outputTable", conf, false); + options.setMapColumnHive("id=STRING,value2=DECIMAL(13,5),value1=INTEGER,value3=DECIMAL(4,5),value4=VARCHAR(255)"); Map colTypes = new SqlTypeMap(); colTypes.put("id", Types.INTEGER); @@ -224,7 +202,8 @@ public void testUserMappingWithDecimal() throws Exception { colTypes.put("value2", Types.DOUBLE); colTypes.put("value3", Types.FLOAT); colTypes.put("value4", Types.CHAR); - writer.setColumnTypes(colTypes); + + setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes); String createTable = writer.getCreateTableStmt(); @@ -245,37 +224,20 @@ public void testUserMappingWithDecimal() throws Exception { @Test public void testUserMappingFailWhenCantBeApplied() throws Exception { - String[] args = { - "--map-column-hive", "id=STRING,value=INTEGER", - }; - Configuration conf = new Configuration(); - SqoopOptions options = - new ImportTool().parseArguments(args, null, null, false); - TableDefWriter writer = new TableDefWriter(options, - null, HsqldbTestServer.getTableName(), "outputTable", conf, false); + options.setMapColumnHive("id=STRING,value=INTEGER"); Map colTypes = new SqlTypeMap(); colTypes.put("id", Types.INTEGER); - writer.setColumnTypes(colTypes); + setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes); thrown.expect(IllegalArgumentException.class); thrown.reportMissingExceptionWithMessage("Expected IllegalArgumentException on non applied Hive type mapping"); - String createTable = writer.getCreateTableStmt(); + writer.getCreateTableStmt(); } @Test public void testHiveDatabase() throws Exception { - String[] args = { - "--hive-database", "db", - }; - Configuration conf = new Configuration(); - SqoopOptions options = - new ImportTool().parseArguments(args, null, null, false); - TableDefWriter writer = new TableDefWriter(options, - null, HsqldbTestServer.getTableName(), "outputTable", conf, false); - - Map colTypes = new SqlTypeMap(); - writer.setColumnTypes(colTypes); + options.setHiveDatabaseName("db"); String createTable = writer.getCreateTableStmt(); assertNotNull(createTable); @@ -286,4 +248,9 @@ public void testHiveDatabase() throws Exception { assertTrue(createTable.contains("`db`.`outputTable`")); } + private void setUpMockConnManager(String tableName, Map typeMap) { + when(connManager.getColumnTypes(tableName)).thenReturn(typeMap); + when(connManager.getColumnNames(tableName)).thenReturn(typeMap.keySet().toArray(new String[]{})); + } + }