SQOOP-3331: Add Mainframe FTP integration test for GDG dataset.

(Chris Teoh via Szabolcs Vasas)

build.xml

@@ -246,6 +246,13 @@
   <property name="sqoop.test.netezza.db.name" value="SQOOP" />
   <property name="sqoop.test.netezza.table.name" value="EMPNZ" />
 
+  <property name="sqoop.test.mainframe.ftp.host" value="localhost" />
+  <property name="sqoop.test.mainframe.ftp.port" value="2121" />
+  <property name="sqoop.test.mainframe.ftp.username" value="test" />
+  <property name="sqoop.test.mainframe.ftp.password" value="test" />
+  <property name="sqoop.test.mainframe.ftp.dataset.gdg" value="TSODIQ1.GDGTEXT" />
+  <property name="sqoop.test.mainframe.ftp.dataset.gdg.filename" value="G0001V43" />
+  <property name="sqoop.test.mainframe.ftp.dataset.gdg.md5" value="f0d0d171fdb8a03dbc1266ed179d7093" />
 
   <condition property="windows">
     <os family="windows" />
@@ -873,6 +880,14 @@
       <sysproperty key="sqoop.test.netezza.db.name" value="${sqoop.test.netezza.db.name}" />
       <sysproperty key="sqoop.test.netezza.table.name" value="${sqoop.test.netezza.table.name}" />
 
+      <sysproperty key="sqoop.test.mainframe.ftp.host" value="${sqoop.test.mainframe.ftp.host}" />
+      <sysproperty key="sqoop.test.mainframe.ftp.port" value="${sqoop.test.mainframe.ftp.port}" />
+      <sysproperty key="sqoop.test.mainframe.ftp.username" value="${sqoop.test.mainframe.ftp.username}" />
+      <sysproperty key="sqoop.test.mainframe.ftp.password" value="${sqoop.test.mainframe.ftp.password}" />
+      <sysproperty key="sqoop.test.mainframe.ftp.dataset.gdg" value="${sqoop.test.mainframe.ftp.dataset.gdg}" />
+      <sysproperty key="sqoop.test.mainframe.ftp.dataset.gdg.filename" value="${sqoop.test.mainframe.ftp.dataset.gdg.filename}" />
+      <sysproperty key="sqoop.test.mainframe.ftp.dataset.gdg.md5" value="${sqoop.test.mainframe.ftp.dataset.gdg.md5}" />
+
       <!-- Location of Hive logs -->
       <!--<sysproperty key="hive.log.dir"
                    value="${test.build.data}/sqoop/logs"/> -->

src/java/org/apache/sqoop/manager/MainframeManager.java

@@ -46,7 +46,6 @@
  */
 public class MainframeManager extends org.apache.sqoop.manager.ConnManager {
   public static final String DEFAULT_DATASET_COLUMN_NAME = "DEFAULT_COLUMN";
-  protected SqoopOptions options;
   private static final Log LOG
       = LogFactory.getLog(MainframeManager.class.getName());
 

src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml

@@ -111,3 +111,14 @@ services:
       interval: 30s
       timeout: 10s
       retries: 20
+  mainframe:
+    image: cntroversycubed/sqoopgdg:afdf57b15d8e71eb77d24d606b77e185ef39ceb3
+    container_name: sqoop_mainframe_gdg_container
+    ports:
+      - 2121:2121
+      - 60000-60009:60000-60009
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "2121"]
+      interval: 30s
+      timeout: 10s
+      retries: 20

src/test/org/apache/sqoop/manager/mainframe/MainframeManagerImportTest.java

@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.mainframe;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Set;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.sqoop.mapreduce.mainframe.MainframeConfiguration;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.tool.MainframeImportTool;
+import org.apache.sqoop.util.FileListing;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Mainframe integration tests
+ * Can be run using -Dtestcase=MainframeManagerImportTest or -Dthirdparty=true.
+ * To run tests:-
+ * Start docker containers by running start-thirdpartytest-db-containers.sh
+ * Run test using ant test -Dtestcase=MainframeManagerImportTest or -Dthirdparty=true
+ * Clean up containers by running stop-thirdpartytest-db-containers.sh
+ * The following properties can be overridden from command line
+ * by setting -D<property.name>=<value>:-
+ * <property name="sqoop.test.mainframe.ftp.host" value="localhost" />
+ * <property name="sqoop.test.mainframe.ftp.port" value="2121" />
+ * <property name="sqoop.test.mainframe.ftp.username" value="test" />
+ * <property name="sqoop.test.mainframe.ftp.password" value="test" />
+ * <property name="sqoop.test.mainframe.ftp.dataset.gdg" value="TSODIQ1.FOLDER" />
+ * <property name="sqoop.test.mainframe.ftp.dataset.gdg.filename" value="G0001V43" />
+ * <property name="sqoop.test.mainframe.ftp.dataset.gdg.md5" value="43eefbe34e466dd3f65a3e867a60809a" />
+ */
+
+public class MainframeManagerImportTest extends ImportJobTestCase {
+  private static final Log LOG = LogFactory.getLog(
+      MainframeManagerImportTest.class.getName());
+
+  @Override
+  protected boolean useHsqldbTestServer() {
+    return false;
+  }
+
+  /** Does the import and verify
+   * @param datasetName the mainframe dataset name
+   * @param datasetType the mainframe dataset type from MainframeConfiguration (s/g/p)
+   * @param fileHashes  each HashMap entry is filename, expected md5sum
+   * @param extraArgs   extra arguments to the tool if required
+   * @throws IOException if it fails to delete the directory or read the file
+   * @throws RuntimeException if it fails to run the mainframe import
+   */
+  private void doImportAndVerify(String datasetName, String datasetType, HashMap<String,String> fileHashes, String ... extraArgs) throws IOException, RuntimeException {
+    Path tablePath = new Path(datasetName);
+
+    File tableFile = new File(tablePath.toString());
+    if (tableFile.exists() && tableFile.isDirectory()) {
+      // remove the directory before running the import.
+      LOG.info(String.format("Removing folder: %s", tableFile));
+      FileListing.recursiveDeleteDir(tableFile);
+    }
+
+    String [] argv = getArgv(datasetName, datasetType, extraArgs);
+    try {
+      MainframeImportTool tool = new MainframeImportTool();
+      runImport(tool,argv);
+    } catch (IOException ioe) {
+      LOG.error("Got IOException during import: " + ioe);
+      throw new RuntimeException(ioe);
+    }
+
+    Set<String> keys = fileHashes.keySet();
+    for (String i : keys) {
+      Path filePath = new Path(tablePath, String.format("%s%s", i,"-m-00000"));
+      LOG.info(String.format("Checking for presence of file: %s with MD5 hash %s",filePath,fileHashes.get(i)));
+      File f = new File(filePath.toString());
+      assertTrue("Could not find imported data file", f.exists());
+      FileInputStream fis = new FileInputStream(f);
+      String md5 = DigestUtils.md5Hex(fis);
+      fis.close();
+      assertTrue(String.format("MD5 sums do not match for file: %s. Got MD5 of %s and expected %s",filePath,md5,fileHashes.get(i)),StringUtils.equalsIgnoreCase(md5, fileHashes.get(i)));
+    }
+  }
+
+  @Test
+  public void testImportGdgText() throws IOException {
+    HashMap<String,String> files = new HashMap<String,String>();
+    files.put(MainframeTestUtil.GDG_DATASET_FILENAME, MainframeTestUtil.EXPECTED_GDG_DATASET_MD5);
+    doImportAndVerify(MainframeTestUtil.GDG_DATASET_NAME, MainframeConfiguration.MAINFRAME_INPUT_DATASET_TYPE_GDG, files);
+  }
+
+  private String [] getArgv(String datasetName, String datasetType, String ... extraArgs) {
+    ArrayList<String> args = new ArrayList<String>();
+
+    CommonArgs.addHadoopFlags(args);
+
+    args.add("--connect");
+    args.add(String.format("%s:%s", MainframeTestUtil.HOST, MainframeTestUtil.PORT));
+    args.add("--username");
+    args.add(MainframeTestUtil.USERNAME);
+    args.add("--password");
+    args.add(MainframeTestUtil.PASSWORD);
+    args.add("--dataset");
+    args.add(datasetName);
+    args.add("--datasettype");
+    args.add(datasetType);
+
+    if (extraArgs.length > 0) {
+      args.add("--");
+      for (String arg : extraArgs) {
+        args.add(arg);
+      }
+    }
+
+    return args.toArray(new String[0]);
+  }
+}

src/test/org/apache/sqoop/manager/mainframe/MainframeTestUtil.java

@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.mainframe;
+
+public class MainframeTestUtil {
+  public static final String HOST = System.getProperty(
+      "sqoop.test.mainframe.ftp.host",
+      "localhost");
+  public static final String PORT = System.getProperty(
+      "sqoop.test.mainframe.ftp.port",
+      "2121");
+  public static final String USERNAME = System.getProperty(
+      "sqoop.test.mainframe.ftp.username",
+      "test");
+  public static final String PASSWORD = System.getProperty(
+      "sqoop.test.mainframe.ftp.password",
+      "test");
+  public static final String GDG_DATASET_NAME = System.getProperty(
+      "sqoop.test.mainframe.ftp.dataset.gdg",
+      "TSODIQ1.GDGTEXT");
+  public static final String GDG_DATASET_FILENAME = System.getProperty(
+      "sqoop.test.mainframe.ftp.dataset.gdg.filename",
+      "G0001V43"
+      );
+  public static final String EXPECTED_GDG_DATASET_MD5 = System.getProperty(
+      "sqoop.test.mainframe.ftp.dataset.gdg.md5",
+      "f0d0d171fdb8a03dbc1266ed179d7093");
+}