Repository: sqoop Updated Branches: refs/heads/trunk 18212bece -> 71523079b
SQOOP-3327: Mainframe FTP needs to Include "Migrated" datasets when parsing the FTP list (Chris Teoh via Szabolcs Vasas) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/71523079 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/71523079 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/71523079 Branch: refs/heads/trunk Commit: 71523079bc61061867ced9b6a597150a3c72a964 Parents: 18212be Author: Szabolcs Vasas <va...@apache.org> Authored: Thu Oct 11 07:52:33 2018 +0200 Committer: Szabolcs Vasas <va...@apache.org> Committed: Thu Oct 11 07:52:33 2018 +0200 ---------------------------------------------------------------------- .../mainframe/MainframeFTPFileEntryParser.java | 36 ++++++++++++++++++-- .../TestMainframeFTPFileEntryParser.java | 12 +++++++ 2 files changed, 46 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/71523079/src/java/org/apache/sqoop/mapreduce/mainframe/MainframeFTPFileEntryParser.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/mainframe/MainframeFTPFileEntryParser.java b/src/java/org/apache/sqoop/mapreduce/mainframe/MainframeFTPFileEntryParser.java index f0b8786..f528a0e 100644 --- a/src/java/org/apache/sqoop/mapreduce/mainframe/MainframeFTPFileEntryParser.java +++ b/src/java/org/apache/sqoop/mapreduce/mainframe/MainframeFTPFileEntryParser.java @@ -33,6 +33,16 @@ public class MainframeFTPFileEntryParser extends ConfigurableFTPFileEntryParserI * xxx300 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOADED * x31167 Tape UNLOAD.EDH.UNLOADT * xxx305 3390 2016/05/23 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD1 + * Migrated DEV.DATA + * Migrated DUMMY.DATA + * OVR343 3390 2018/01/23 1 1 FB 132 27984 PS EMPTY + * Migrated JCL.CNTL + * OVR346 3390 2018/01/22 1 1 FB 80 27920 PS MIXED.FB80 + * Migrated PLAIN.FB80 + * OVR341 3390 2018/01/23 1 9 VA 125 129 PS PRDA.SPFLOG1.LIST + * G20427 Tape UNLOAD.ABCDE.ZZ9UYT.FB.TAPE + * SEM352 3390 2018/01/23 1 1 FB 150 1500 PS USER.BRODCAST + * OVR346 3390 2018/01/23 3 3 FB 80 6160 PO USER.ISPPROF */ // match Unit and Dsname @@ -41,6 +51,8 @@ public class MainframeFTPFileEntryParser extends ConfigurableFTPFileEntryParserI private static String NON_TAPE_REGEX = "^\\S+\\s+(\\S+)\\s+.*?(\\d+)\\s+(\\S+)\\s+(\\S+)$"; static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; //= "MMM d yyyy"; //Nov 9 2001 + // match Volume, DsName + private static String MIGRATED_REGEX = "^(Migrated)\\s+(\\S+)$"; static final String DEFAULT_RECENT_DATE_FORMAT = "MMM d HH:mm"; //Nov 9 20:06 @@ -51,7 +63,8 @@ public class MainframeFTPFileEntryParser extends ConfigurableFTPFileEntryParserI private static String dsOrgPDSExtendedString = "PO-E"; private static String dsOrgSeqString = "PS"; private static Pattern nonTapePattern = Pattern.compile(NON_TAPE_REGEX); - + private static final String MIGRATED_STRING = "Migrated"; + private static Pattern migratedPattern = Pattern.compile(MIGRATED_REGEX); private static final Log LOG = LogFactory.getLog(MainframeFTPFileEntryParser.class.getName()); public MainframeFTPFileEntryParser() { @@ -100,7 +113,7 @@ public class MainframeFTPFileEntryParser extends ConfigurableFTPFileEntryParserI } return file; } - return null; + return parseMigratedEntry(entry); } @Override @@ -108,4 +121,23 @@ public class MainframeFTPFileEntryParser extends ConfigurableFTPFileEntryParserI return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null, null, null, null); } + + private FTPFile parseMigratedEntry(String entry) { + // check for Migrated dataset + Matcher m = migratedPattern.matcher(entry); + if (m.matches()) { + // Volume = "Migrated" + String volume = m.group(1); + String dsName = m.group(2); + FTPFile file = new FTPFile(); + if (MIGRATED_STRING.equals(volume)) { + file.setRawListing(entry); + file.setType(FTPFile.FILE_TYPE); + file.setName(dsName); + LOG.info(String.format("Migrated Dataset found: %s, %s", volume, dsName)); + return file; + } + } + return null; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/71523079/src/test/org/apache/sqoop/mapreduce/mainframe/TestMainframeFTPFileEntryParser.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/mapreduce/mainframe/TestMainframeFTPFileEntryParser.java b/src/test/org/apache/sqoop/mapreduce/mainframe/TestMainframeFTPFileEntryParser.java index eb0f8c0..26ed961 100644 --- a/src/test/org/apache/sqoop/mapreduce/mainframe/TestMainframeFTPFileEntryParser.java +++ b/src/test/org/apache/sqoop/mapreduce/mainframe/TestMainframeFTPFileEntryParser.java @@ -39,6 +39,15 @@ x31167 Tape xxx305 3390 2016/05/23 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD1 xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD2 xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD3 +Migrated DUMMY.DATA +OVR343 3390 2018/01/23 1 1 FB 132 27984 PS EMPTY +Migrated JCL.CNTL +OVR346 3390 2018/01/22 1 1 FB 80 27920 PS MIXED.FB80 +Migrated PLAIN.FB80 +OVR341 3390 2018/01/23 1 9 VA 125 129 PS PRDA.SPFLOG1.LIST +G20427 Tape UNLOAD.ABCDE.ZZ9UYT.FB.TAPE +SEM352 3390 2018/01/23 1 1 FB 150 1500 PS USER.BRODCAST +OVR346 3390 2018/01/23 3 3 FB 80 6160 PO USER.ISPPROF */ listing = new ArrayList<String>(); listing.add("Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname"); @@ -47,6 +56,9 @@ xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD3 listing.add("xxx305 3390 2016/05/23 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD1"); listing.add("xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD2"); listing.add("xxx305 3390 2016/05/25 1 45 VB 2349 27998 PS UNLOAD.EDH.UNLOAD3"); + listing.add("Migrated PLAIN.FB80"); + listing.add("Migrated DUMMY.DATA"); + listing.add("Migrated JCL.CNTL"); } @AfterClass
