1 /* 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with 4 * this work for additional information regarding copyright ownership. 5 * The ASF licenses this file to You under the Apache License, Version 2.0 6 * (the "License"); you may not use this file except in compliance with 7 * the License. You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18 package org.apache.commons.net.ftp.parser; 19 20 import java.text.ParseException; 21 import java.util.List; 22 23 import org.apache.commons.net.ftp.FTPClientConfig; 24 import org.apache.commons.net.ftp.FTPFile; 25 26 /** 27 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS 28 * Systems. 29 * 30 * @author <a href="henrik.sorensen@balcab.ch">Henrik Sorensen</a> 31 * @author <a href="jnadler@srcginc.com">Jeff Nadler</a> 32 * @author <a href="wnoto@openfinance.com">William Noto</a> 33 * 34 * @version $Id: MVSFTPEntryParser.java 1490237 2013-06-06 11:17:23Z sebb $ 35 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for 36 * usage instructions) 37 */ 38 public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl { 39 40 static final int UNKNOWN_LIST_TYPE = -1; 41 static final int FILE_LIST_TYPE = 0; 42 static final int MEMBER_LIST_TYPE = 1; 43 static final int UNIX_LIST_TYPE = 2; 44 static final int JES_LEVEL_1_LIST_TYPE = 3; 45 static final int JES_LEVEL_2_LIST_TYPE = 4; 46 47 private int isType = UNKNOWN_LIST_TYPE; 48 49 /** 50 * Fallback parser for Unix-style listings 51 */ 52 private UnixFTPEntryParser unixFTPEntryParser; 53 54 /** 55 * Dates are ignored for file lists, but are used for member lists where 56 * possible 57 */ 58 static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18 59 // 13:52 60 61 /** 62 * Matches these entries: Volume Unit Referred Ext Used Recfm Lrecl BlkSz 63 * Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK 64 * 65 */ 66 static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume 67 // ignored 68 "\\S+\\s+" + // unit - ignored 69 "\\S+\\s+" + // access date - ignored 70 "\\S+\\s+" + // extents -ignored 71 "\\S+\\s+" + // used - ignored 72 "[FV]\\S*\\s+" + // recfm - must start with F or V 73 "\\S+\\s+" + // logical record length -ignored 74 "\\S+\\s+" + // block size - ignored 75 "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist 76 // but only support: PS, PO, PO-E 77 "(\\S+)\\s*"; // Dataset Name (file name) 78 79 /** 80 * Matches these entries: Name VV.MM Created Changed Size Init Mod Id 81 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 82 */ 83 static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name 84 "\\S+\\s+" + // version, modification (ignored) 85 "\\S+\\s+" + // create date (ignored) 86 "(\\S+)\\s+" + // modification date 87 "(\\S+)\\s+" + // modification time 88 "\\S+\\s+" + // size in lines (ignored) 89 "\\S+\\s+" + // size in lines at creation(ignored) 90 "\\S+\\s+" + // lines modified (ignored) 91 "\\S+\\s*"; // id of user who modified (ignored) 92 93 /** 94 * Matches these entries, note: no header: IBMUSER1 JOB01906 OUTPUT 3 Spool 95 * Files 012345678901234567890123456789012345678901234 1 2 3 4 96 */ 97 static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job 98 // name 99 // ignored 100 "(\\S+)\\s+" + // job number 101 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) 102 "(\\S+)\\s+" + // number of spool files 103 "(\\S+)\\s+" + // Text "Spool" ignored 104 "(\\S+)\\s*" // Text "Files" ignored 105 ; 106 107 /** 108 * JES INTERFACE LEVEL 2 parser Matches these entries: JOBNAME JOBID OWNER 109 * STATUS CLASS IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 110 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 111 * 012345678901234567890123456789012345678901234 1 2 3 4 112 * 012345678901234567890123456789012345678901234567890 113 */ 114 115 static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job 116 // name 117 // ignored 118 "(\\S+)\\s+" + // job number 119 "(\\S+)\\s+" + // owner ignored 120 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored 121 "(\\S+)\\s+" + // job class ignored 122 "(\\S+).*" // rest ignored 123 ; 124 125 /* 126 * --------------------------------------------------------------------- 127 * Very brief and incomplete description of the zOS/MVS-filesystem. (Note: 128 * "zOS" is the operating system on the mainframe, and is the new name for 129 * MVS) 130 * 131 * The filesystem on the mainframe does not have hierarchal structure as for 132 * example the unix filesystem. For a more comprehensive description, please 133 * refer to the IBM manuals 134 * 135 * @LINK: 136 * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS 137 * 138 * 139 * Dataset names ============= 140 * 141 * A dataset name consist of a number of qualifiers separated by '.', each 142 * qualifier can be at most 8 characters, and the total length of a dataset 143 * can be max 44 characters including the dots. 144 * 145 * 146 * Dataset organisation ==================== 147 * 148 * A dataset represents a piece of storage allocated on one or more disks. 149 * The structure of the storage is described with the field dataset 150 * organinsation (DSORG). There are a number of dataset organisations, but 151 * only two are usable for FTP transfer. 152 * 153 * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: 154 * extended partitioned dataset 155 * 156 * The PS file is just a flat file, as you would find it on the unix file 157 * system. 158 * 159 * The PO and PO-E files, can be compared to a single level directory 160 * structure. A PO file consist of a number of dataset members, or files if 161 * you will. It is possible to CD into the file, and to retrieve the 162 * individual members. 163 * 164 * 165 * Dataset record format ===================== 166 * 167 * The physical layout of the dataset is described on the dataset itself. 168 * There are a number of record formats (RECFM), but just a few is relavant 169 * for the FTP transfer. 170 * 171 * Any one beginning with either F or V can safely used by FTP transfer. All 172 * others should only be used with great care, so this version will just 173 * ignore the other record formats. F means a fixed number of records per 174 * allocated storage, and V means a variable number of records. 175 * 176 * 177 * Other notes =========== 178 * 179 * The file system supports automatically backup and retrieval of datasets. 180 * If a file is backed up, the ftp LIST command will return: ARCIVE Not 181 * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST 182 * 183 * 184 * Implementation notes ==================== 185 * 186 * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning 187 * with F or V, is fully parsed. 188 * 189 * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. 190 * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name 191 * FTPFile.Timestamp: change time or null 192 * 193 * 194 * 195 * Additional information ====================== 196 * 197 * The MVS ftp server supports a number of features via the FTP interface. 198 * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2> 199 * SEQ is the default and used for normal file transfer JES is used to 200 * interact with the Job Entry Subsystem (JES) similar to a job scheduler 201 * DB2 is used to interact with a DB2 subsystem 202 * 203 * This parser supports SEQ and JES. 204 * 205 * 206 * 207 * 208 * 209 * 210 */ 211 212 /** 213 * The sole constructor for a MVSFTPEntryParser object. 214 * 215 */ 216 public MVSFTPEntryParser() { 217 super(""); // note the regex is set in preParse. 218 super.configure(null); // configure parser with default configurations 219 } 220 221 /** 222 * Parses a line of an z/OS - MVS FTP server file listing and converts it 223 * into a usable format in the form of an <code> FTPFile </code> instance. 224 * If the file listing line doesn't describe a file, then 225 * <code> null </code> is returned. Otherwise a <code> FTPFile </code> 226 * instance representing the file is returned. 227 * 228 * @param entry 229 * A line of text from the file listing 230 * @return An FTPFile instance corresponding to the supplied entry 231 */ 232 // @Override 233 public FTPFile parseFTPEntry(String entry) { 234 boolean isParsed = false; 235 FTPFile f = new FTPFile(); 236 237 if (isType == FILE_LIST_TYPE) { 238 isParsed = parseFileList(f, entry); 239 } else if (isType == MEMBER_LIST_TYPE) { 240 isParsed = parseMemberList(f, entry); 241 if (!isParsed) { 242 isParsed = parseSimpleEntry(f, entry); 243 } 244 } else if (isType == UNIX_LIST_TYPE) { 245 isParsed = parseUnixList(f, entry); 246 } else if (isType == JES_LEVEL_1_LIST_TYPE) { 247 isParsed = parseJeslevel1List(f, entry); 248 } else if (isType == JES_LEVEL_2_LIST_TYPE) { 249 isParsed = parseJeslevel2List(f, entry); 250 } 251 252 if (!isParsed) { 253 f = null; 254 } 255 256 return f; 257 } 258 259 /** 260 * Parse entries representing a dataset list. Only datasets with DSORG PS or 261 * PO or PO-E and with RECFM F* or V* will be parsed. 262 * 263 * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred 264 * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80 265 * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device 266 * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO 267 * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB 268 * 269 * ----------------------------------- Group within Regex [1] Volume [2] 270 * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record 271 * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: 272 * Dataset organisation. Many exists but only support: PS, PO, PO-E [10] 273 * Dsname: Dataset name 274 * 275 * Note: When volume is ARCIVE, it means the dataset is stored somewhere in 276 * a tape archive. These entries is currently not supported by this parser. 277 * A null value is returned. 278 * 279 * @param file 280 * will be updated with Name, Type, Timestamp if parsed. 281 * @param entry zosDirectoryEntry 282 * @return true: entry was parsed, false: entry was not parsed. 283 */ 284 private boolean parseFileList(FTPFile file, String entry) { 285 if (matches(entry)) { 286 file.setRawListing(entry); 287 String name = group(2); 288 String dsorg = group(1); 289 file.setName(name); 290 291 // DSORG 292 if ("PS".equals(dsorg)) { 293 file.setType(FTPFile.FILE_TYPE); 294 } 295 else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) { 296 // regex already ruled out anything other than PO or PO-E 297 file.setType(FTPFile.DIRECTORY_TYPE); 298 } 299 else { 300 return false; 301 } 302 303 return true; 304 } 305 306 return false; 307 } 308 309 /** 310 * Parse entries within a partitioned dataset. 311 * 312 * Format of a memberlist within a PDS: 1 2 3 4 5 6 7 8 9 Name VV.MM Created 313 * Changed Size Init Mod Id TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 314 * 0 KIL001 TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001 315 * 316 * ------------------------------------------- [1] Name [2] VV.MM: Version . 317 * modification [3] Created: yyyy / MM / dd [4,5] Changed: yyyy / MM / dd 318 * HH:mm [6] Size: number of lines [7] Init: number of lines when first 319 * created [8] Mod: number of modified lines a last save [9] Id: User id for 320 * last update 321 * 322 * 323 * @param file 324 * will be updated with Name, Type and Timestamp if parsed. 325 * @param entry zosDirectoryEntry 326 * @return true: entry was parsed, false: entry was not parsed. 327 */ 328 private boolean parseMemberList(FTPFile file, String entry) { 329 if (matches(entry)) { 330 file.setRawListing(entry); 331 String name = group(1); 332 String datestr = group(2) + " " + group(3); 333 file.setName(name); 334 file.setType(FTPFile.FILE_TYPE); 335 try { 336 file.setTimestamp(super.parseTimestamp(datestr)); 337 } catch (ParseException e) { 338 e.printStackTrace(); 339 // just ignore parsing errors. 340 // TODO check this is ok 341 return false; // this is a parsing failure too. 342 } 343 return true; 344 } 345 346 return false; 347 } 348 349 /** 350 * Assigns the name to the first word of the entry. Only to be used from a 351 * safe context, for example from a memberlist, where the regex for some 352 * reason fails. Then just assign the name field of FTPFile. 353 * 354 * @param file 355 * @param entry 356 * @return true if the entry string is non-null and non-empty 357 */ 358 private boolean parseSimpleEntry(FTPFile file, String entry) { 359 if (entry != null && entry.trim().length() > 0) { 360 file.setRawListing(entry); 361 String name = entry.split(" ")[0]; 362 file.setName(name); 363 file.setType(FTPFile.FILE_TYPE); 364 return true; 365 } 366 return false; 367 } 368 369 /** 370 * Parse the entry as a standard unix file. Using the UnixFTPEntryParser. 371 * 372 * @param file 373 * @param entry 374 * @return true: entry is parsed, false: entry could not be parsed. 375 */ 376 private boolean parseUnixList(FTPFile file, String entry) { 377 file = unixFTPEntryParser.parseFTPEntry(entry); 378 if (file == null) { 379 return false; 380 } 381 return true; 382 } 383 384 /** 385 * Matches these entries, note: no header: [1] [2] [3] [4] [5] IBMUSER1 386 * JOB01906 OUTPUT 3 Spool Files 387 * 012345678901234567890123456789012345678901234 1 2 3 4 388 * ------------------------------------------- Group in regex [1] Job name 389 * [2] Job number [3] Job status (INPUT,ACTIVE,OUTPUT) [4] Number of sysout 390 * files [5] The string "Spool Files" 391 * 392 * 393 * @param file 394 * will be updated with Name, Type and Timestamp if parsed. 395 * @param entry zosDirectoryEntry 396 * @return true: entry was parsed, false: entry was not parsed. 397 */ 398 private boolean parseJeslevel1List(FTPFile file, String entry) { 399 if (matches(entry)) { 400 if (group(3).equalsIgnoreCase("OUTPUT")) { 401 file.setRawListing(entry); 402 String name = group(2); /* Job Number, used by GET */ 403 file.setName(name); 404 file.setType(FTPFile.FILE_TYPE); 405 return true; 406 } 407 } 408 409 return false; 410 } 411 412 /** 413 * Matches these entries, note: no header: [1] [2] [3] [4] [5] JOBNAME JOBID 414 * OWNER STATUS CLASS IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool 415 * files IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 416 * 012345678901234567890123456789012345678901234 1 2 3 4 417 * ------------------------------------------- Group in regex [1] Job name 418 * [2] Job number [3] Owner [4] Job status (INPUT,ACTIVE,OUTPUT) [5] Job 419 * Class [6] The rest 420 * 421 * 422 * @param file 423 * will be updated with Name, Type and Timestamp if parsed. 424 * @param entry zosDirectoryEntry 425 * @return true: entry was parsed, false: entry was not parsed. 426 */ 427 private boolean parseJeslevel2List(FTPFile file, String entry) { 428 if (matches(entry)) { 429 if (group(4).equalsIgnoreCase("OUTPUT")) { 430 file.setRawListing(entry); 431 String name = group(2); /* Job Number, used by GET */ 432 file.setName(name); 433 file.setType(FTPFile.FILE_TYPE); 434 return true; 435 } 436 } 437 438 return false; 439 } 440 441 /** 442 * preParse is called as part of the interface. Per definition is is called 443 * before the parsing takes place. Three kind of lists is recognize: 444 * z/OS-MVS File lists z/OS-MVS Member lists unix file lists 445 * @since 2.0 446 */ 447 @Override 448 public List<String> preParse(List<String> orig) { 449 // simply remove the header line. Composite logic will take care of the 450 // two different types of 451 // list in short order. 452 if (orig != null && orig.size() > 0) { 453 String header = orig.get(0); 454 if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) { 455 setType(FILE_LIST_TYPE); 456 super.setRegex(FILE_LIST_REGEX); 457 } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) { 458 setType(MEMBER_LIST_TYPE); 459 super.setRegex(MEMBER_LIST_REGEX); 460 } else if (header.indexOf("total") == 0) { 461 setType(UNIX_LIST_TYPE); 462 unixFTPEntryParser = new UnixFTPEntryParser(); 463 } else if (header.indexOf("Spool Files") >= 30) { 464 setType(JES_LEVEL_1_LIST_TYPE); 465 super.setRegex(JES_LEVEL_1_LIST_REGEX); 466 } else if (header.indexOf("JOBNAME") == 0 467 && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS 468 setType(JES_LEVEL_2_LIST_TYPE); 469 super.setRegex(JES_LEVEL_2_LIST_REGEX); 470 } else { 471 setType(UNKNOWN_LIST_TYPE); 472 } 473 474 if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary 475 orig.remove(0); 476 } 477 } 478 479 return orig; 480 } 481 482 /** 483 * Explicitly set the type of listing being processed. 484 * @param type The listing type. 485 */ 486 void setType(int type) { 487 isType = type; 488 } 489 490 /* 491 * @return 492 */ 493 @Override 494 protected FTPClientConfig getDefaultConfiguration() { 495 return new FTPClientConfig(FTPClientConfig.SYST_MVS, 496 DEFAULT_DATE_FORMAT, null, null, null, null); 497 } 498 499 }