diff --git a/build.xml b/build.xml
index a85705fa..0ae729bc 100644
--- a/build.xml
+++ b/build.xml
@@ -246,6 +246,13 @@
+
+
+
+
+
+
+
@@ -873,6 +880,14 @@
+
+
+
+
+
+
+
+
diff --git a/src/java/org/apache/sqoop/manager/MainframeManager.java b/src/java/org/apache/sqoop/manager/MainframeManager.java
index 4e8be155..70b9fc94 100644
--- a/src/java/org/apache/sqoop/manager/MainframeManager.java
+++ b/src/java/org/apache/sqoop/manager/MainframeManager.java
@@ -46,7 +46,6 @@
*/
public class MainframeManager extends org.apache.sqoop.manager.ConnManager {
public static final String DEFAULT_DATASET_COLUMN_NAME = "DEFAULT_COLUMN";
- protected SqoopOptions options;
private static final Log LOG
= LogFactory.getLog(MainframeManager.class.getName());
diff --git a/src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml b/src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml
index 2f4a07f6..4648f545 100644
--- a/src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml
+++ b/src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml
@@ -111,3 +111,14 @@ services:
interval: 30s
timeout: 10s
retries: 20
+ mainframe:
+ image: cntroversycubed/sqoopgdg:afdf57b15d8e71eb77d24d606b77e185ef39ceb3
+ container_name: sqoop_mainframe_gdg_container
+ ports:
+ - 2121:2121
+ - 60000-60009:60000-60009
+ healthcheck:
+ test: ["CMD", "nc", "-z", "localhost", "2121"]
+ interval: 30s
+ timeout: 10s
+ retries: 20
diff --git a/src/test/org/apache/sqoop/manager/mainframe/MainframeManagerImportTest.java b/src/test/org/apache/sqoop/manager/mainframe/MainframeManagerImportTest.java
new file mode 100644
index 00000000..041dfb78
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/mainframe/MainframeManagerImportTest.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.mainframe;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Set;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.sqoop.mapreduce.mainframe.MainframeConfiguration;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.tool.MainframeImportTool;
+import org.apache.sqoop.util.FileListing;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Mainframe integration tests
+ * Can be run using -Dtestcase=MainframeManagerImportTest or -Dthirdparty=true.
+ * To run tests:-
+ * Start docker containers by running start-thirdpartytest-db-containers.sh
+ * Run test using ant test -Dtestcase=MainframeManagerImportTest or -Dthirdparty=true
+ * Clean up containers by running stop-thirdpartytest-db-containers.sh
+ * The following properties can be overridden from command line
+ * by setting -D=:-
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ */
+
+public class MainframeManagerImportTest extends ImportJobTestCase {
+ private static final Log LOG = LogFactory.getLog(
+ MainframeManagerImportTest.class.getName());
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ /** Does the import and verify
+ * @param datasetName the mainframe dataset name
+ * @param datasetType the mainframe dataset type from MainframeConfiguration (s/g/p)
+ * @param fileHashes each HashMap entry is filename, expected md5sum
+ * @param extraArgs extra arguments to the tool if required
+ * @throws IOException if it fails to delete the directory or read the file
+ * @throws RuntimeException if it fails to run the mainframe import
+ */
+ private void doImportAndVerify(String datasetName, String datasetType, HashMap fileHashes, String ... extraArgs) throws IOException, RuntimeException {
+ Path tablePath = new Path(datasetName);
+
+ File tableFile = new File(tablePath.toString());
+ if (tableFile.exists() && tableFile.isDirectory()) {
+ // remove the directory before running the import.
+ LOG.info(String.format("Removing folder: %s", tableFile));
+ FileListing.recursiveDeleteDir(tableFile);
+ }
+
+ String [] argv = getArgv(datasetName, datasetType, extraArgs);
+ try {
+ MainframeImportTool tool = new MainframeImportTool();
+ runImport(tool,argv);
+ } catch (IOException ioe) {
+ LOG.error("Got IOException during import: " + ioe);
+ throw new RuntimeException(ioe);
+ }
+
+ Set keys = fileHashes.keySet();
+ for (String i : keys) {
+ Path filePath = new Path(tablePath, String.format("%s%s", i,"-m-00000"));
+ LOG.info(String.format("Checking for presence of file: %s with MD5 hash %s",filePath,fileHashes.get(i)));
+ File f = new File(filePath.toString());
+ assertTrue("Could not find imported data file", f.exists());
+ FileInputStream fis = new FileInputStream(f);
+ String md5 = DigestUtils.md5Hex(fis);
+ fis.close();
+ assertTrue(String.format("MD5 sums do not match for file: %s. Got MD5 of %s and expected %s",filePath,md5,fileHashes.get(i)),StringUtils.equalsIgnoreCase(md5, fileHashes.get(i)));
+ }
+ }
+
+ @Test
+ public void testImportGdgText() throws IOException {
+ HashMap files = new HashMap();
+ files.put(MainframeTestUtil.GDG_DATASET_FILENAME, MainframeTestUtil.EXPECTED_GDG_DATASET_MD5);
+ doImportAndVerify(MainframeTestUtil.GDG_DATASET_NAME, MainframeConfiguration.MAINFRAME_INPUT_DATASET_TYPE_GDG, files);
+ }
+
+ private String [] getArgv(String datasetName, String datasetType, String ... extraArgs) {
+ ArrayList args = new ArrayList();
+
+ CommonArgs.addHadoopFlags(args);
+
+ args.add("--connect");
+ args.add(String.format("%s:%s", MainframeTestUtil.HOST, MainframeTestUtil.PORT));
+ args.add("--username");
+ args.add(MainframeTestUtil.USERNAME);
+ args.add("--password");
+ args.add(MainframeTestUtil.PASSWORD);
+ args.add("--dataset");
+ args.add(datasetName);
+ args.add("--datasettype");
+ args.add(datasetType);
+
+ if (extraArgs.length > 0) {
+ args.add("--");
+ for (String arg : extraArgs) {
+ args.add(arg);
+ }
+ }
+
+ return args.toArray(new String[0]);
+ }
+}
diff --git a/src/test/org/apache/sqoop/manager/mainframe/MainframeTestUtil.java b/src/test/org/apache/sqoop/manager/mainframe/MainframeTestUtil.java
new file mode 100644
index 00000000..f28ff36c
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/mainframe/MainframeTestUtil.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.mainframe;
+
+public class MainframeTestUtil {
+ public static final String HOST = System.getProperty(
+ "sqoop.test.mainframe.ftp.host",
+ "localhost");
+ public static final String PORT = System.getProperty(
+ "sqoop.test.mainframe.ftp.port",
+ "2121");
+ public static final String USERNAME = System.getProperty(
+ "sqoop.test.mainframe.ftp.username",
+ "test");
+ public static final String PASSWORD = System.getProperty(
+ "sqoop.test.mainframe.ftp.password",
+ "test");
+ public static final String GDG_DATASET_NAME = System.getProperty(
+ "sqoop.test.mainframe.ftp.dataset.gdg",
+ "TSODIQ1.GDGTEXT");
+ public static final String GDG_DATASET_FILENAME = System.getProperty(
+ "sqoop.test.mainframe.ftp.dataset.gdg.filename",
+ "G0001V43"
+ );
+ public static final String EXPECTED_GDG_DATASET_MD5 = System.getProperty(
+ "sqoop.test.mainframe.ftp.dataset.gdg.md5",
+ "f0d0d171fdb8a03dbc1266ed179d7093");
+}