github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-03 04:42:22 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3327: Mainframe FTP needs to Include "Migrated" datasets when parsing the FTP list

Browse Source 
(Chris Teoh via Szabolcs Vasas)
This commit is contained in:
Szabolcs Vasas 2018-10-11 07:52:33 +02:00
parent 18212becec
commit 71523079bc
2 changed files with 46 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/java/org/apache/sqoop/mapreduce/mainframe/MainframeFTPFileEntryParser.java
View File

@ -33,6 +33,16 @@ public class MainframeFTPFileEntryParser extends ConfigurableFTPFileEntryParserI
* xxx300 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOADED * xxx300 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOADED
* x31167 Tape UNLOAD.EDH.UNLOADT * x31167 Tape UNLOAD.EDH.UNLOADT
* xxx305 3390 2016/05/23 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD1 * xxx305 3390 2016/05/23 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD1
* Migrated DEV.DATA
* Migrated DUMMY.DATA
* OVR343 3390 2018/01/23 1 1 FB 132 27984 PS EMPTY
* Migrated JCL.CNTL
* OVR346 3390 2018/01/22 1 1 FB 80 27920 PS MIXED.FB80
* Migrated PLAIN.FB80
* OVR341 3390 2018/01/23 1 9 VA 125 129 PS PRDA.SPFLOG1.LIST
* G20427 Tape UNLOAD.ABCDE.ZZ9UYT.FB.TAPE
* SEM352 3390 2018/01/23 1 1 FB 150 1500 PS USER.BRODCAST
* OVR346 3390 2018/01/23 3 3 FB 80 6160 PO USER.ISPPROF
*/ */
// match Unit and Dsname // match Unit and Dsname
@ -41,6 +51,8 @@ public class MainframeFTPFileEntryParser extends ConfigurableFTPFileEntryParserI
private static String NON_TAPE_REGEX = "^\\S+\\s+(\\S+)\\s+.*?(\\d+)\\s+(\\S+)\\s+(\\S+)$"; private static String NON_TAPE_REGEX = "^\\S+\\s+(\\S+)\\s+.*?(\\d+)\\s+(\\S+)\\s+(\\S+)$";
static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm";
//= "MMM d yyyy"; //Nov 9 2001 //= "MMM d yyyy"; //Nov 9 2001
// match Volume, DsName
private static String MIGRATED_REGEX = "^(Migrated)\\s+(\\S+)$";
static final String DEFAULT_RECENT_DATE_FORMAT = "MMM d HH:mm"; //Nov 9 20:06 static final String DEFAULT_RECENT_DATE_FORMAT = "MMM d HH:mm"; //Nov 9 20:06
@ -51,7 +63,8 @@ public class MainframeFTPFileEntryParser extends ConfigurableFTPFileEntryParserI
private static String dsOrgPDSExtendedString = "PO-E"; private static String dsOrgPDSExtendedString = "PO-E";
private static String dsOrgSeqString = "PS"; private static String dsOrgSeqString = "PS";
private static Pattern nonTapePattern = Pattern.compile(NON_TAPE_REGEX); private static Pattern nonTapePattern = Pattern.compile(NON_TAPE_REGEX);
private static final String MIGRATED_STRING = "Migrated";
private static Pattern migratedPattern = Pattern.compile(MIGRATED_REGEX);
private static final Log LOG = LogFactory.getLog(MainframeFTPFileEntryParser.class.getName()); private static final Log LOG = LogFactory.getLog(MainframeFTPFileEntryParser.class.getName());
public MainframeFTPFileEntryParser() { public MainframeFTPFileEntryParser() {
@ -100,7 +113,7 @@ public FTPFile parseFTPEntry(String entry) {
} }
return file; return file;
} }
return null; return parseMigratedEntry(entry);
} }
@Override @Override
@ -108,4 +121,23 @@ protected FTPClientConfig getDefaultConfiguration() {
return new FTPClientConfig(FTPClientConfig.SYST_MVS, return new FTPClientConfig(FTPClientConfig.SYST_MVS,
DEFAULT_DATE_FORMAT, null, null, null, null); DEFAULT_DATE_FORMAT, null, null, null, null);
} }
private FTPFile parseMigratedEntry(String entry) {
// check for Migrated dataset
Matcher m = migratedPattern.matcher(entry);
if (m.matches()) {
// Volume = "Migrated"
String volume = m.group(1);
String dsName = m.group(2);
FTPFile file = new FTPFile();
if (MIGRATED_STRING.equals(volume)) {
file.setRawListing(entry);
file.setType(FTPFile.FILE_TYPE);
file.setName(dsName);
LOG.info(String.format("Migrated Dataset found: %s, %s", volume, dsName));
return file;
}
}
return null;
}
} }

12
src/test/org/apache/sqoop/mapreduce/mainframe/TestMainframeFTPFileEntryParser.java
View File

@ -39,6 +39,15 @@ public static void setUpBeforeClass() throws Exception {
xxx305 3390 2016/05/23 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD1 xxx305 3390 2016/05/23 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD1
xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD2 xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD2
xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD3 xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD3
Migrated DUMMY.DATA
OVR343 3390 2018/01/23 1 1 FB 132 27984 PS EMPTY
Migrated JCL.CNTL
OVR346 3390 2018/01/22 1 1 FB 80 27920 PS MIXED.FB80
Migrated PLAIN.FB80
OVR341 3390 2018/01/23 1 9 VA 125 129 PS PRDA.SPFLOG1.LIST
G20427 Tape UNLOAD.ABCDE.ZZ9UYT.FB.TAPE
SEM352 3390 2018/01/23 1 1 FB 150 1500 PS USER.BRODCAST
OVR346 3390 2018/01/23 3 3 FB 80 6160 PO USER.ISPPROF
*/ */
listing = new ArrayList<String>(); listing = new ArrayList<String>();
listing.add("Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname"); listing.add("Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname");
@ -47,6 +56,9 @@ public static void setUpBeforeClass() throws Exception {
listing.add("xxx305 3390 2016/05/23 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD1"); listing.add("xxx305 3390 2016/05/23 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD1");
listing.add("xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD2"); listing.add("xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD2");
listing.add("xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD3"); listing.add("xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD3");
listing.add("Migrated PLAIN.FB80");
listing.add("Migrated DUMMY.DATA");
listing.add("Migrated JCL.CNTL");
} }
@AfterClass @AfterClass