github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 22:02:31 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3169: Evaluate and fix SQLServer Manual tests

Browse Source 
(Boglarka Egyed via Anna Szonyi)
This commit is contained in:
Anna Szonyi 2017-04-13 11:33:45 +02:00
parent afd6a8610e
commit 0ca73d4e71
18 changed files with 432 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
build.xml
View File

@ -224,6 +224,9 @@
<property name="sqoop.test.sqlserver.connectstring.host_url"
value="jdbc:sqlserver://sqlserverhost:1433"/>
<property name="sqoop.test.sqlserver.database"
value="sqooptest"/>
<property name="java.security.krb5.realm"
value="OX.AC.UK"/>
@ -850,6 +853,9 @@
<sysproperty key="sqoop.test.sqlserver.connectstring.host_url"
value="${sqoop.test.sqlserver.connectstring.host_url}"/>
<sysproperty key="sqoop.test.sqlserver.database"
value="${sqoop.test.sqlserver.database}"/>
<sysproperty key="sqoop.test.msserver.connector.factory"
value="${sqoop.test.msserver.connector.factory}"/>

12
src/test/com/cloudera/sqoop/hive/TestHiveImport.java
View File

@ -104,7 +104,7 @@ protected void setNumCols(int numCols) {
setColNames(cols);
}
protected String[] getTypesNewLineTest() {
protected String[] getTypes() {
String[] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" };
return types;
}
@ -295,7 +295,7 @@ public void testNormalHiveImportAsParquet() throws IOException {
final String TABLE_NAME = "NORMAL_HIVE_IMPORT_AS_PARQUET";
setCurTableName(TABLE_NAME);
setNumCols(3);
String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" };
String [] types = getTypes();
String [] vals = { "'test'", "42", "'somestring'" };
String [] extraArgs = {"--as-parquetfile"};
@ -382,7 +382,7 @@ public void testCreateOverwriteHiveImportAsParquet() throws IOException {
final String TABLE_NAME = "CREATE_OVERWRITE_HIVE_IMPORT_AS_PARQUET";
setCurTableName(TABLE_NAME);
setNumCols(3);
String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" };
String [] types = getTypes();
String [] vals = { "'test'", "42", "'somestring'" };
String [] extraArgs = {"--as-parquetfile"};
ImportTool tool = new ImportTool();
@ -441,7 +441,7 @@ public void testAppendHiveImportAsParquet() throws IOException {
final String TABLE_NAME = "APPEND_HIVE_IMPORT_AS_PARQUET";
setCurTableName(TABLE_NAME);
setNumCols(3);
String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" };
String [] types = getTypes();
String [] vals = { "'test'", "42", "'somestring'" };
String [] extraArgs = {"--as-parquetfile"};
String [] args = getArgv(false, extraArgs);
@ -560,7 +560,7 @@ public void testFieldWithHiveDelims() throws IOException,
LOG.info("Doing import of single row into FIELD_WITH_NL_HIVE_IMPORT table");
setCurTableName(TABLE_NAME);
setNumCols(3);
String[] types = getTypesNewLineTest();
String[] types = getTypes();
String[] vals = { "'test with \n new lines \n'", "42",
"'oh no " + '\01' + " field delims " + '\01' + "'", };
String[] moreArgs = { "--"+ BaseSqoopTool.HIVE_DROP_DELIMS_ARG };
@ -609,7 +609,7 @@ public void testFieldWithHiveDelimsReplacement() throws IOException,
+ "FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT table");
setCurTableName(TABLE_NAME);
setNumCols(3);
String[] types = getTypesNewLineTest();
String[] types = getTypes();
String[] vals = { "'test with\nnew lines\n'", "42",
"'oh no " + '\01' + " field delims " + '\01' + "'", };
String[] moreArgs = { "--"+BaseSqoopTool.HIVE_DELIMS_REPLACEMENT_ARG, " "};

31
src/test/com/cloudera/sqoop/manager/SQLServerManagerExportManualTest.java
View File

@ -53,10 +53,16 @@ public class SQLServerManagerExportManualTest extends ExportJobTestCase {
static final String HOST_URL = System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
static final String DATABASE_NAME = System.getProperty(
"sqoop.test.sqlserver.database",
"sqooptest");
static final String DATABASE_USER = System.getProperty(
"ms.sqlserver.username",
"sqoopuser");
static final String DATABASE_PASSWORD = System.getProperty(
"ms.sqlserver.password",
"password");
static final String DATABASE_NAME = "SQOOPTEST";
static final String DATABASE_USER = "SQOOPUSER";
static final String DATABASE_PASSWORD = "PASSWORD";
static final String SCHEMA_DBO = "dbo";
static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL";
static final String DBO_BINARY_TABLE_NAME = "BINARYTYPE_MSSQL";
@ -84,6 +90,11 @@ protected boolean useHsqldbTestServer() {
return false;
}
private String getDropTableStatement(String schema, String tableName) {
return "DROP TABLE IF EXISTS " + manager.escapeObjectName(schema)
+ "." + manager.escapeObjectName(tableName);
}
@Before
public void setUp() {
super.setUp();
@ -247,6 +258,14 @@ public void createSQLServerBinaryTypeTable(String schema, String table) {
@After
public void tearDown() {
try {
Statement stmt = conn.createStatement();
stmt.executeUpdate(getDropTableStatement(SCHEMA_DBO, DBO_TABLE_NAME));
stmt.executeUpdate(getDropTableStatement(SCHEMA_SCH, SCH_TABLE_NAME));
} catch (SQLException e) {
LOG.error("Can't clean up the database:", e);
}
super.tearDown();
try {
conn.close();
@ -382,11 +401,11 @@ public void testUpsertTextExport() throws IOException, SQLException {
});
// first time will be insert.
runExport(getArgv(SCH_TABLE_NAME, "--update-key", "id",
"--update-mode", "allowinsert"));
"--update-mode", "allowinsert", "--", "--schema", SCHEMA_SCH));
// second time will be update.
runExport(getArgv(SCH_TABLE_NAME, "--update-key", "id",
"--update-mode", "allowinsert"));
assertRowCount(2, escapeObjectName(SCH_TABLE_NAME), conn);
"--update-mode", "allowinsert", "--", "--schema", SCHEMA_SCH));
assertRowCount(2, escapeObjectName(SCHEMA_SCH) + "." + escapeObjectName(SCH_TABLE_NAME), conn);
}
public static void checkSQLBinaryTableContent(String[] expected, String tableName, Connection connection){

27
src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java
View File

@ -75,10 +75,16 @@ public class SQLServerManagerImportManualTest extends ImportJobTestCase {
static final String HOST_URL = System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
static final String DATABASE_NAME = System.getProperty(
"sqoop.test.sqlserver.database",
"sqooptest");
static final String DATABASE_USER = System.getProperty(
"ms.sqlserver.username",
"sqoopuser");
static final String DATABASE_PASSWORD = System.getProperty(
"ms.sqlserver.password",
"password");
static final String DATABASE_NAME = "SQOOPTEST";
static final String DATABASE_USER = "SQOOPUSER";
static final String DATABASE_PASSWORD = "PASSWORD";
static final String SCHEMA_DBO = "dbo";
static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL";
static final String SCHEMA_SCH = "sch";
@ -94,6 +100,7 @@ public class SQLServerManagerImportManualTest extends ImportJobTestCase {
private SQLServerManager manager;
private Configuration conf = new Configuration();
private Connection conn = null;
@Override
protected Configuration getConf() {
@ -105,6 +112,11 @@ protected boolean useHsqldbTestServer() {
return false;
}
private String getDropTableStatement(String schema, String tableName) {
return "DROP TABLE IF EXISTS " + manager.escapeObjectName(schema)
+ "." + manager.escapeObjectName(tableName);
}
@Before
public void setUp() {
super.setUp();
@ -130,7 +142,6 @@ public void createTableAndPopulateData(String schema, String table) {
String fulltableName = manager.escapeObjectName(schema)
+ "." + manager.escapeObjectName(table);
Connection conn = null;
Statement stmt = null;
// Create schema if needed
@ -208,6 +219,14 @@ public void createTableAndPopulateData(String schema, String table) {
@After
public void tearDown() {
try {
Statement stmt = conn.createStatement();
stmt.executeUpdate(getDropTableStatement(SCHEMA_DBO, DBO_TABLE_NAME));
stmt.executeUpdate(getDropTableStatement(SCHEMA_SCH, SCH_TABLE_NAME));
} catch (SQLException e) {
LOG.error("Can't clean up the database:", e);
}
super.tearDown();
try {
manager.close();

21
src/test/org/apache/sqoop/manager/sqlserver/MSSQLTestUtils.java
View File

@ -17,9 +17,6 @@
*/
package org.apache.sqoop.manager.sqlserver;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
@ -27,9 +24,6 @@
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -47,10 +41,15 @@ public class MSSQLTestUtils {
"ms.sqlserver.username", "SQOOPUSER");
static final String DATABASE_PASSWORD = System.getProperty(
"ms.sqlserver.password", "PASSWORD");
static final String DATABASE_NAME = System.getProperty(
"sqoop.test.sqlserver.database",
"sqooptest");
public static final String HOST_URL = System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
public static final String CONNECT_STRING = HOST_URL + ";database=" + DATABASE_NAME;
public static final String CREATE_TALBE_LINEITEM
= "CREATE TABLE TPCH1M_LINEITEM"
+ "( [L_ORDERKEY] [int] NULL, [L_PARTKEY] "
@ -70,7 +69,7 @@ private Connection getConnection() {
if (conn == null) {
try {
Connection con = DriverManager.getConnection(HOST_URL,
Connection con = DriverManager.getConnection(CONNECT_STRING,
DATABASE_USER, DATABASE_PASSWORD);
conn = con;
return con;
@ -158,6 +157,14 @@ public static String getDBPassWord() {
return DATABASE_PASSWORD;
}
public static String getDBDatabaseName() {
return DATABASE_NAME;
}
public static String getDBConnectString() {
return CONNECT_STRING;
}
public void dropTableIfExists(String table) throws SQLException {
conn = getConnection();
System.out.println("Dropping table : " + table);

118
src/test/org/apache/sqoop/manager/sqlserver/ManagerCompatExport.java
View File

@ -40,7 +40,6 @@
import com.cloudera.sqoop.tool.ExportTool;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.fail;
@ -117,162 +116,162 @@ public abstract void createFile(DATATYPES dt, String[] data)
public abstract void createFile(DATATYPES dt, String data) throws Exception;
@Test
public void testVarBinary() {
public void testVarBinary() throws Exception {
exportTestMethod(DATATYPES.VARBINARY);
}
@Test
public void testTime() {
public void testTime() throws Exception {
exportTestMethod(DATATYPES.TIME);
}
@Test
public void testSmalldatetime() {
public void testSmalldatetime() throws Exception {
exportTestMethod(DATATYPES.SMALLDATETIME);
}
@Test
public void testdatetime2() {
public void testdatetime2() throws Exception {
exportTestMethod(DATATYPES.DATETIME2);
}
@Test
public void testdatetime() {
public void testdatetime() throws Exception {
exportTestMethod(DATATYPES.DATETIME);
}
@Test
public void testdatetimeoffset() {
public void testdatetimeoffset() throws Exception {
exportTestMethod(DATATYPES.DATETIMEOFFSET);
}
@Test
public void testDecimal() {
public void testDecimal() throws Exception {
exportTestMethod(DATATYPES.DECIMAL);
}
@Test
public void testNumeric() {
public void testNumeric() throws Exception {
exportTestMethod(DATATYPES.NUMERIC);
}
@Test
public void testBigInt() {
public void testBigInt() throws Exception {
exportTestMethod(DATATYPES.BIGINT);
}
@Test
public void testInt() {
public void testInt() throws Exception {
exportTestMethod(DATATYPES.INT);
}
@Test
public void testSmallInt() {
public void testSmallInt() throws Exception {
exportTestMethod(DATATYPES.SMALLINT);
}
@Test
public void testTinyint() {
public void testTinyint() throws Exception {
exportTestMethod(DATATYPES.TINYINT);
}
@Test
public void testFloat() {
public void testFloat() throws Exception {
exportTestMethod(DATATYPES.FLOAT);
}
@Test
public void testReal() {
public void testReal() throws Exception {
exportTestMethod(DATATYPES.REAL);
}
@Test
public void testDate() {
public void testDate() throws Exception {
exportTestMethod(DATATYPES.DATE);
}
@Test
public void testMoney() {
public void testMoney() throws Exception {
exportTestMethod(DATATYPES.MONEY);
}
@Test
public void testSmallMoney() {
public void testSmallMoney() throws Exception {
exportTestMethod(DATATYPES.SMALLMONEY);
}
@Test
public void testText() {
public void testText() throws Exception {
exportTestMethod(DATATYPES.TEXT);
}
@Test
public void testVarchar() {
public void testVarchar() throws Exception {
exportTestMethod(DATATYPES.VARCHAR);
}
@Test
public void testChar() {
public void testChar() throws Exception {
exportTestMethod(DATATYPES.CHAR);
}
@Test
public void testNText() {
public void testNText() throws Exception {
exportTestMethod(DATATYPES.NTEXT);
}
@Test
public void testNChar() {
public void testNChar() throws Exception {
exportTestMethod(DATATYPES.NCHAR);
}
@Test
public void testNVarchar() {
public void testNVarchar() throws Exception {
exportTestMethod(DATATYPES.NVARCHAR);
}
@Test
public void testImage() {
public void testImage() throws Exception {
exportTestMethod(DATATYPES.IMAGE);
}
@Test
public void testBinary() {
public void testBinary() throws Exception {
exportTestMethod(DATATYPES.BINARY);
}
public void exportTestMethod(DATATYPES dt) {
public void exportTestMethod(DATATYPES dt) throws SQLException {
int exceptionCount = 0;
List testdata = tdfs.getTestdata(dt);
@ -370,6 +369,8 @@ public void exportTestMethod(DATATYPES dt) {
} catch (Error e) {
addToReport(current, e);
exceptionCount++;
} finally {
dropTableIfExists(getTableName(dt));
}
}
if (exceptionCount > 0) {
@ -377,7 +378,7 @@ public void exportTestMethod(DATATYPES dt) {
System.out.println("There were failures for :" + dt.toString());
System.out.println("Failed for " + exceptionCount + "/"
+ testdata.size() + " test data samples\n");
System.out.println("Sroll up for detailed errors");
System.out.println("Scroll up for detailed errors");
System.out
.println("----------------------------------------------------------"
+ "-");
@ -387,25 +388,6 @@ public void exportTestMethod(DATATYPES dt) {
}
/*
*/
public String[] extractData(List data, String negPosFlag,
KEY_STRINGS readBackType) {
List<String> filtered = new ArrayList<String>();
for (Iterator<MSSQLTestData> itr = data.iterator(); itr.hasNext();) {
MSSQLTestData current = itr.next();
if (current.getData(KEY_STRINGS.NEG_POS_FLAG).toString().equals(
negPosFlag)) {
filtered.add(current.getData(readBackType));
}
}
String[] ret = new String[filtered.size()];
filtered.toArray(ret);
return ret;
}
public void verifyExport(DATATYPES dt, String[] data) throws SQLException {
LOG.info("Verifying export: " + getTableName());
// Check that we got back the correct number of records.
@ -446,43 +428,6 @@ public void verifyExport(DATATYPES dt, String[] data) throws SQLException {
}
public void verifyNegativeExport(DATATYPES dt, String[] data)
throws SQLException {
LOG.info("Verifying export: " + getTableName());
// Check that we got back the correct number of records.
Connection conn = getManager().getConnection();
PreparedStatement statement = conn.prepareStatement("SELECT "
+ getColName() + " FROM " + getTableName(dt),
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
System.out.println("data samples being compared : " + data.length);
ResultSet rs = null;
try {
rs = statement.executeQuery();
int cnt = 0;
try {
while (rs.next()) {
String tmp = rs.getString(1);
String expected = data[cnt++];
System.out.println("Readback, expected" + tmp + " :"
+ expected);
if (tmp == null) {
assertNull("Must be null", expected);
} else {
assertNotSame("Data must match", expected, tmp);
}
}
} finally {
rs.close();
}
} finally {
statement.close();
}
}
public void verifyExport(DATATYPES dt, String data) throws SQLException {
verifyExport(dt, new String[] { data });
@ -568,10 +513,9 @@ protected boolean useHsqldbTestServer() {
return false;
}
@Override
protected String getConnectString() {
return System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
return MSSQLTestUtils.getDBConnectString();
}
/**

25
src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileManualTest.java
View File

@ -22,7 +22,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
import java.io.OutputStream;
@ -30,11 +29,29 @@
import java.io.BufferedWriter;
/**
* Export delimited file SQL Server.
*/
* Test to export delimited file to SQL Server.
*
* This uses JDBC to export data to an SQLServer database from HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerDatatypeExportDelimitedFileManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerDatatypeExportDelimitedFileManualTest
extends ManagerCompatExport {
@Override
public void createFile(DATATYPES dt, String[] data) throws IOException {
Path tablePath = getTablePath(dt);
Path filePath = new Path(tablePath, "part0000");
@ -60,10 +77,12 @@ public void createFile(DATATYPES dt, String[] data) throws IOException {
os.close();
}
@Override
public void createFile(DATATYPES dt, String data) throws IOException {
createFile(dt, new String[] { data });
}
@Override
public String getOutputFileName() {
return "ManagerCompatExportDelim.txt";
}

32
src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileManualTest.java
View File

@ -29,7 +29,6 @@
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
import com.cloudera.sqoop.SqoopOptions;
@ -43,14 +42,31 @@
import static org.junit.Assert.assertTrue;
/**
* Export sequence file to SQL Server test.
*/
* Test to export sequence file to SQL Server.
*
* This uses JDBC to export data to an SQLServer database from HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerDatatypeExportSequenceFileManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerDatatypeExportSequenceFileManualTest
extends ManagerCompatExport {
private static Map jars = new HashMap();
@Override
@Override
public void createFile(DATATYPES dt, String[] data) throws Exception {
try {
codeGen(dt);
@ -142,11 +158,9 @@ public String[] codeGen(DATATYPES dt) throws Exception {
jars.put(dt, jarFileName);
return (getArgv(dt, "--class-name", className, "--jar-file",
jarFileName));
}
@Override
protected String[] getArgv(DATATYPES dt) {
String[] args = super.getArgv(dt);
@ -177,7 +191,7 @@ protected String[] getCodeGenArgv(DATATYPES dt) {
codeGenArgv.add("--table");
codeGenArgv.add(getTableName(dt));
codeGenArgv.add("--connect");
codeGenArgv.add(getConnectString());
codeGenArgv.add(MSSQLTestUtils.getDBConnectString());
codeGenArgv.add("--fields-terminated-by");
codeGenArgv.add("\\t");
codeGenArgv.add("--lines-terminated-by");
@ -225,7 +239,7 @@ protected String[] getArgv(DATATYPES dt, String... additionalArgv) {
args.add("--export-dir");
args.add(getTablePath(dt).toString());
args.add("--connect");
args.add(getConnectString());
args.add(MSSQLTestUtils.getDBConnectString());
args.add("--fields-terminated-by");
args.add("\\t");
args.add("--lines-terminated-by");

88
src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java
View File

@ -30,7 +30,6 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
import com.cloudera.sqoop.Sqoop;
import com.cloudera.sqoop.SqoopOptions;
@ -38,13 +37,31 @@
import com.cloudera.sqoop.testutil.CommonArgs;
import com.cloudera.sqoop.tool.ImportTool;
import com.cloudera.sqoop.util.ClassLoaderStack;
import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/**
* Test import delimited file from SQL Server.
* Test to import delimited file from SQL Server.
*
* This uses JDBC to import data from an SQLServer database to HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerDatatypeImportDelimitedFileManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerDatatypeImportDelimitedFileManualTest
extends SQLServerDatatypeImportSequenceFileManualTest {
@ -88,7 +105,7 @@ protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
args.add("--warehouse-dir");
args.add(getWarehouseDir());
args.add("--connect");
args.add(getConnectString());
args.add(MSSQLTestUtils.getDBConnectString());
args.add("--num-mappers");
args.add("2");
@ -190,13 +207,13 @@ protected void verifyImport(String expectedVal, String[] importCols) {
} finally {
IOUtils.closeStream(reader);
}
LOG.info("Read back from sequencefile: " + line);
LOG.info("Read back from delimited file: " + line);
foundRecord = true;
// Add trailing '\n' to expected value since
// SqoopRecord.toString()
// encodes the record delim.
if (null == expectedVal) {
assertEquals("Error validating result from SeqFile",
assertEquals("Error validating result from delimited file",
"null\n", line);
}
} catch (EOFException eoe) {
@ -209,7 +226,7 @@ protected void verifyImport(String expectedVal, String[] importCols) {
}
if (!foundRecord) {
fail("Couldn't read any records from SequenceFiles");
fail("Couldn't read any records from delimited file");
}
} catch (IOException ioe) {
LOG.error(StringUtils.stringifyException(ioe));
@ -221,15 +238,6 @@ protected void verifyImport(String expectedVal, String[] importCols) {
}
}
@Test
public void testVarBinary() {
if (!supportsVarBinary()) {
return;
}
dataTypeTest(DATATYPES.VARBINARY);
}
@Test
public void testTime() {
if (!supportsTime()) {
@ -240,6 +248,56 @@ public void testTime() {
dataTypeTest(DATATYPES.TIME);
}
@Ignore("Ignored as used type is not supported for table splitting.")
@Test
public void testVarBinary() {
}
@Ignore("Ignored as used type is not supported for table splitting.")
@Test
public void testBit() {
}
@Ignore("Ignored as used type is not supported for table splitting.")
@Test
public void testBit2() {
}
@Ignore("Ignored as used type is not supported for table splitting.")
@Test
public void testBit3() {
}
@Ignore("Ignored as used type is not supported for table splitting.")
@Test
public void testNChar() {
}
@Ignore("Ignored as used type is not supported for table splitting.")
@Test
public void testChar() {
}
@Ignore("Ignored as used type is not supported for table splitting.")
@Test
public void testVarchar() {
}
@Ignore("Ignored as used type is not supported for table splitting.")
@Test
public void testNVarchar() {
}
@Ignore("Ignored as used type is not supported for table splitting.")
@Test
public void testBinary() {
}
@Ignore("Ignored as used type is not supported for table splitting.")
@Test
public void testTimestamp3() {
}
public String getResportFileName(){
return this.getClass().toString()+".txt";
}

23
src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java
View File

@ -42,7 +42,24 @@
import static org.junit.Assert.fail;
/**
* Testing import of a sequence file to SQL Server.
* Test importing sequence file from SQL Server.
*
* This uses JDBC to import data from an SQLServer database to HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerDatatypeImportSequenceFileManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerDatatypeImportSequenceFileManualTest extends
ManagerCompatTestCase {
@ -89,9 +106,7 @@ protected boolean useHsqldbTestServer() {
}
protected String getConnectString() {
return System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
return MSSQLTestUtils.getDBConnectString();
}
/**

41
src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java
View File

@ -23,22 +23,37 @@
import java.sql.SQLException;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.hive.TestHiveImport;
import com.cloudera.sqoop.testutil.CommonArgs;
import com.cloudera.sqoop.tool.SqoopTool;
import org.junit.After;
import org.junit.Before;
import static org.junit.Assert.fail;
/**
* Test import to Hive from SQL Server.
*
* This uses JDBC to import data from an SQLServer database to HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerHiveImportManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerHiveImportManualTest extends TestHiveImport {
@ -47,18 +62,28 @@ public void setUp() {
super.setUp();
}
@After
public void tearDown() {
try {
dropTableIfExists(getTableName());
} catch (SQLException sqle) {
LOG.info("Table clean-up failed: " + sqle);
} finally {
super.tearDown();
}
}
protected boolean useHsqldbTestServer() {
return false;
}
protected String getConnectString() {
return System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
return MSSQLTestUtils.getDBConnectString();
}
//SQL Server pads out
protected String[] getTypesNewLineTest() {
@Override
protected String[] getTypes() {
String[] types = { "VARCHAR(32)", "INTEGER", "VARCHAR(64)" };
return types;
}
@ -115,7 +140,7 @@ SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
protected String[] getArgv(boolean includeHadoopFlags, String[] moreArgs) {
ArrayList<String> args = new ArrayList<String>();
System.out.println("Ovverdien getArgv is called..");
System.out.println("Overridden getArgv is called..");
if (includeHadoopFlags) {
CommonArgs.addHadoopFlags(args);
}

24
src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java
View File

@ -30,7 +30,6 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -49,8 +48,25 @@
/**
* Test methods of the generic SqlManager implementation.
*
* This uses JDBC to import data from an SQLServer database to HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerManagerManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerManagerManualTest {
public class SQLServerManagerManualTest {
public static final Log LOG = LogFactory.getLog(
SQLServerManagerManualTest.class.getName());
@ -282,9 +298,7 @@ protected boolean useHsqldbTestServer() {
}
protected String getConnectString() {
return System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
return MSSQLTestUtils.getDBConnectString();
}
/**

36
src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsManualTest.java
View File

@ -22,14 +22,31 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.TestMultiCols;
import org.junit.After;
import org.junit.Test;
/**
* Test multiple columns SQL Server.
* Test multiple columns in SQL Server.
*
* This uses JDBC to import data from an SQLServer database to HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerMultiColsManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerMultiColsManualTest extends TestMultiCols {
@ -38,9 +55,7 @@ protected boolean useHsqldbTestServer() {
}
protected String getConnectString() {
return System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
return MSSQLTestUtils.getDBConnectString();
}
/**
@ -77,6 +92,17 @@ protected SqoopOptions getSqoopOptions(Configuration conf) {
}
@After
public void tearDown() {
try {
dropTableIfExists(getTableName());
} catch (SQLException sqle) {
LOG.info("Table clean-up failed: " + sqle);
} finally {
super.tearDown();
}
}
@Test
public void testMixed4() {
// Overridden to bypass test case invalid for MSSQL server

26
src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsManualTest.java
View File

@ -26,8 +26,6 @@
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@ -37,7 +35,6 @@
import org.apache.hadoop.mapred.Utils;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
@ -56,7 +53,24 @@
import static org.junit.Assert.fail;
/**
* Test that using multiple mapper splits works.
* Test multiple mapper splits in SQL Server.
*
* This uses JDBC to import data from an SQLServer database to HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerMultiMapsManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerMultiMapsManualTest extends ImportJobTestCase {
@ -247,9 +261,7 @@ protected boolean useHsqldbTestServer() {
}
protected String getConnectString() {
return System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
return MSSQLTestUtils.getDBConnectString();
}
/**

38
src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsManualTest.java
View File

@ -25,8 +25,6 @@
import java.sql.SQLException;
import java.util.ArrayList;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@ -37,7 +35,6 @@
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
@ -48,6 +45,7 @@
import com.cloudera.sqoop.testutil.ReparseMapper;
import com.cloudera.sqoop.tool.ImportTool;
import com.cloudera.sqoop.util.ClassLoaderStack;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -55,7 +53,24 @@
/**
* Test that the parse() methods generated in user SqoopRecord implementations
* work.
* work in SQL Server.
*
* This uses JDBC to import data from an SQLServer database to HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerParseMethodsManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerParseMethodsManualTest extends ImportJobTestCase {
@ -72,6 +87,17 @@ public void setUp() {
}
}
@After
public void tearDown() {
try {
dropTableIfExists(getTableName());
} catch (SQLException sqle) {
LOG.info("Table clean-up failed: " + sqle);
} finally {
super.tearDown();
}
}
/**
* Create the argv to pass to Sqoop.
*
@ -220,9 +246,7 @@ protected boolean useHsqldbTestServer() {
}
protected String getConnectString() {
return System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
return MSSQLTestUtils.getDBConnectString();
}
/**

26
src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryManualTest.java
View File

@ -25,14 +25,11 @@
import java.sql.SQLException;
import java.util.ArrayList;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
@ -50,7 +47,24 @@
import static org.junit.Assert.fail;
/**
* Test that --query works in Sqoop.
* Test that --query works in SQL Server.
*
* This uses JDBC to import data from an SQLServer database to HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerQueryManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerQueryManualTest extends ImportJobTestCase {
@ -244,9 +258,7 @@ protected boolean useHsqldbTestServer() {
}
protected String getConnectString() {
return System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
return MSSQLTestUtils.getDBConnectString();
}
/**

26
src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByManualTest.java
View File

@ -25,14 +25,11 @@
import java.sql.SQLException;
import java.util.ArrayList;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
@ -50,7 +47,24 @@
import static org.junit.Assert.fail;
/**
* Test that --split-by works.
* Test that --split-by works in SQL Server.
*
* This uses JDBC to import data from an SQLServer database to HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerSplitByManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerSplitByManualTest extends ImportJobTestCase {
@ -213,9 +227,7 @@ protected boolean useHsqldbTestServer() {
}
protected String getConnectString() {
return System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
return MSSQLTestUtils.getDBConnectString();
}
protected String getTableName() {

28
src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereManualTest.java
View File

@ -25,18 +25,14 @@
import java.sql.SQLException;
import java.util.ArrayList;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
import com.cloudera.sqoop.manager.JdbcMySQLExportTest;
import com.cloudera.sqoop.orm.CompilationManager;
import com.cloudera.sqoop.testutil.CommonArgs;
import com.cloudera.sqoop.testutil.ImportJobTestCase;
@ -51,8 +47,24 @@
import static org.junit.Assert.fail;
/**
* Test that --where works in Sqoop. Methods essentially copied out of the other
* Test* classes.
* Test that --where works in SQL Server.
*
* This uses JDBC to import data from an SQLServer database to HDFS.
*
* Since this requires an SQLServer installation,
* this class is named in such a way that Sqoop's default QA process does
* not run it. You need to run this manually with
* -Dtestcase=SQLServerWhereManualTest.
*
* You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
* where Sqoop will be able to access it (since this library cannot be checked
* into Apache's tree for licensing reasons).
*
* To set up your test environment:
* Install SQL Server Express 2012
* Create a database SQOOPTEST
* Create a login SQOOPUSER with password PASSWORD and grant all
* access for SQOOPTEST to SQOOPUSER.
*/
public class SQLServerWhereManualTest extends ImportJobTestCase {
@ -235,9 +247,7 @@ protected boolean useHsqldbTestServer() {
}
protected String getConnectString() {
return System.getProperty(
"sqoop.test.sqlserver.connectstring.host_url",
"jdbc:sqlserver://sqlserverhost:1433");
return MSSQLTestUtils.getDBConnectString();
}
/**