github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-02 18:11:13 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3308: Mock ConnManager field in TestTableDefWriter

Browse Source 
(Szabolcs Vasas via Boglarka Egyed)
This commit is contained in:
Boglarka Egyed 2018-03-29 16:09:31 +02:00
parent 7186b9d654
commit d2c40366a7
2 changed files with 54 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/java/org/apache/sqoop/hive/TableDefWriter.java
View File

@ -20,7 +20,6 @@
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
import java.util.Date;
import java.text.DateFormat;
@ -79,17 +78,6 @@ public TableDefWriter(final SqoopOptions opts, final ConnManager connMgr,
this.commentsEnabled = withComments;
}
private Map<String, Integer> externalColTypes;
/**
* Set the column type map to be used.
* (dependency injection for testing; not used in production.)
*/
public void setColumnTypes(Map<String, Integer> colTypes) {
this.externalColTypes = colTypes;
LOG.debug("Using test-controlled type map");
}
/**
* Get the column names to import.
*/
@ -97,14 +85,6 @@ public void setColumnTypes(Map<String, Integer> colTypes) {
String [] colNames = options.getColumns();
if (null != colNames) {
return colNames; // user-specified column names.
} else if (null != externalColTypes) {
// Test-injection column mapping. Extract the col names from this.
ArrayList<String> keyList = new ArrayList<String>();
for (String key : externalColTypes.keySet()) {
keyList.add(key);
}
return keyList.toArray(new String[keyList.size()]);
} else if (null != inputTableName) {
return connManager.getColumnNames(inputTableName);
} else {
@ -119,16 +99,11 @@ public String getCreateTableStmt() throws IOException {
Map<String, Integer> columnTypes;
Properties userMapping = options.getMapColumnHive();
Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir());
if (externalColTypes != null) {
// Use pre-defined column types.
columnTypes = externalColTypes;
// Get these from the database.
if (null != inputTableName) {
columnTypes = connManager.getColumnTypes(inputTableName);
} else {
// Get these from the database.
if (null != inputTableName) {
columnTypes = connManager.getColumnTypes(inputTableName);
} else {
columnTypes = connManager.getColumnTypesForQuery(options.getSqlQuery());
}
columnTypes = connManager.getColumnTypesForQuery(options.getSqlQuery());
}
String [] colNames = getColumnNames();

133
src/test/org/apache/sqoop/hive/TestTableDefWriter.java
View File

@ -23,12 +23,13 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.manager.ConnManager;
import org.apache.sqoop.util.SqlTypeMap;
import org.apache.sqoop.SqoopOptions;
import org.apache.sqoop.tool.ImportTool;
import org.apache.sqoop.testutil.HsqldbTestServer;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@ -39,6 +40,9 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
@ -49,9 +53,34 @@ public class TestTableDefWriter {
public static final Log LOG = LogFactory.getLog(
TestTableDefWriter.class.getName());
private ConnManager connManager;
private Configuration conf;
private SqoopOptions options;
private TableDefWriter writer;
private String inputTable;
private String outputTable;
@Rule
public ExpectedException thrown = ExpectedException.none();
@Before
public void before() {
inputTable = HsqldbTestServer.getTableName();
outputTable = "outputTable";
connManager = mock(ConnManager.class);
conf = new Configuration();
options = new SqoopOptions();
when(connManager.getColumnTypes(anyString())).thenReturn(new SqlTypeMap<String, Integer>());
when(connManager.getColumnNames(anyString())).thenReturn(new String[]{});
writer = new TableDefWriter(options, connManager, inputTable, outputTable, conf, false);
}
// Test getHiveOctalCharCode and expect an IllegalArgumentException.
private void expectExceptionInCharCode(int charCode) {
thrown.expect(IllegalArgumentException.class);
@ -73,14 +102,6 @@ public void testHiveOctalCharCode() {
@Test
public void testDifferentTableNames() throws Exception {
Configuration conf = new Configuration();
SqoopOptions options = new SqoopOptions();
TableDefWriter writer = new TableDefWriter(options, null,
"inputTable", "outputTable", conf, false);
Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
writer.setColumnTypes(colTypes);
String createTable = writer.getCreateTableStmt();
String loadData = writer.getLoadDataStmt();
@ -91,24 +112,15 @@ public void testDifferentTableNames() throws Exception {
assertTrue(createTable.indexOf(
"CREATE TABLE IF NOT EXISTS `outputTable`") != -1);
assertTrue(loadData.indexOf("INTO TABLE `outputTable`") != -1);
assertTrue(loadData.indexOf("/inputTable'") != -1);
assertTrue(loadData.indexOf("/" + inputTable + "'") != -1);
}
@Test
public void testDifferentTargetDirs() throws Exception {
String targetDir = "targetDir";
String inputTable = "inputTable";
String outputTable = "outputTable";
Configuration conf = new Configuration();
SqoopOptions options = new SqoopOptions();
// Specify a different target dir from input table name
options.setTargetDir(targetDir);
TableDefWriter writer = new TableDefWriter(options, null,
inputTable, outputTable, conf, false);
Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
writer.setColumnTypes(colTypes);
String createTable = writer.getCreateTableStmt();
String loadData = writer.getLoadDataStmt();
@ -125,18 +137,8 @@ public void testDifferentTargetDirs() throws Exception {
@Test
public void testPartitions() throws Exception {
String[] args = {
"--hive-partition-key", "ds",
"--hive-partition-value", "20110413",
};
Configuration conf = new Configuration();
SqoopOptions options =
new ImportTool().parseArguments(args, null, null, false);
TableDefWriter writer = new TableDefWriter(options,
null, "inputTable", "outputTable", conf, false);
Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
writer.setColumnTypes(colTypes);
options.setHivePartitionKey("ds");
options.setHivePartitionValue("20110413");
String createTable = writer.getCreateTableStmt();
String loadData = writer.getLoadDataStmt();
@ -152,18 +154,8 @@ public void testPartitions() throws Exception {
@Test
public void testLzoSplitting() throws Exception {
String[] args = {
"--compress",
"--compression-codec", "lzop",
};
Configuration conf = new Configuration();
SqoopOptions options =
new ImportTool().parseArguments(args, null, null, false);
TableDefWriter writer = new TableDefWriter(options,
null, "inputTable", "outputTable", conf, false);
Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
writer.setColumnTypes(colTypes);
options.setUseCompression(true);
options.setCompressionCodec("lzop");
String createTable = writer.getCreateTableStmt();
String loadData = writer.getLoadDataStmt();
@ -181,19 +173,13 @@ public void testLzoSplitting() throws Exception {
@Test
public void testUserMappingNoDecimal() throws Exception {
String[] args = {
"--map-column-hive", "id=STRING,value=INTEGER",
};
Configuration conf = new Configuration();
SqoopOptions options =
new ImportTool().parseArguments(args, null, null, false);
TableDefWriter writer = new TableDefWriter(options,
null, HsqldbTestServer.getTableName(), "outputTable", conf, false);
options.setMapColumnHive("id=STRING,value=INTEGER");
Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
colTypes.put("id", Types.INTEGER);
colTypes.put("value", Types.VARCHAR);
writer.setColumnTypes(colTypes);
setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes);
String createTable = writer.getCreateTableStmt();
@ -208,15 +194,7 @@ public void testUserMappingNoDecimal() throws Exception {
@Test
public void testUserMappingWithDecimal() throws Exception {
String[] args = {
"--map-column-hive", "id=STRING,value2=DECIMAL(13,5),value1=INTEGER," +
"value3=DECIMAL(4,5),value4=VARCHAR(255)",
};
Configuration conf = new Configuration();
SqoopOptions options =
new ImportTool().parseArguments(args, null, null, false);
TableDefWriter writer = new TableDefWriter(options,
null, HsqldbTestServer.getTableName(), "outputTable", conf, false);
options.setMapColumnHive("id=STRING,value2=DECIMAL(13,5),value1=INTEGER,value3=DECIMAL(4,5),value4=VARCHAR(255)");
Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
colTypes.put("id", Types.INTEGER);
@ -224,7 +202,8 @@ public void testUserMappingWithDecimal() throws Exception {
colTypes.put("value2", Types.DOUBLE);
colTypes.put("value3", Types.FLOAT);
colTypes.put("value4", Types.CHAR);
writer.setColumnTypes(colTypes);
setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes);
String createTable = writer.getCreateTableStmt();
@ -245,37 +224,20 @@ public void testUserMappingWithDecimal() throws Exception {
@Test
public void testUserMappingFailWhenCantBeApplied() throws Exception {
String[] args = {
"--map-column-hive", "id=STRING,value=INTEGER",
};
Configuration conf = new Configuration();
SqoopOptions options =
new ImportTool().parseArguments(args, null, null, false);
TableDefWriter writer = new TableDefWriter(options,
null, HsqldbTestServer.getTableName(), "outputTable", conf, false);
options.setMapColumnHive("id=STRING,value=INTEGER");
Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
colTypes.put("id", Types.INTEGER);
writer.setColumnTypes(colTypes);
setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes);
thrown.expect(IllegalArgumentException.class);
thrown.reportMissingExceptionWithMessage("Expected IllegalArgumentException on non applied Hive type mapping");
String createTable = writer.getCreateTableStmt();
writer.getCreateTableStmt();
}
@Test
public void testHiveDatabase() throws Exception {
String[] args = {
"--hive-database", "db",
};
Configuration conf = new Configuration();
SqoopOptions options =
new ImportTool().parseArguments(args, null, null, false);
TableDefWriter writer = new TableDefWriter(options,
null, HsqldbTestServer.getTableName(), "outputTable", conf, false);
Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
writer.setColumnTypes(colTypes);
options.setHiveDatabaseName("db");
String createTable = writer.getCreateTableStmt();
assertNotNull(createTable);
@ -286,4 +248,9 @@ public void testHiveDatabase() throws Exception {
assertTrue(createTable.contains("`db`.`outputTable`"));
}
private void setUpMockConnManager(String tableName, Map<String, Integer> typeMap) {
when(connManager.getColumnTypes(tableName)).thenReturn(typeMap);
when(connManager.getColumnNames(tableName)).thenReturn(typeMap.keySet().toArray(new String[]{}));
}
}