http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/util/ExpressionValidator.java ---------------------------------------------------------------------- diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/util/ExpressionValidator.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/util/ExpressionValidator.java deleted file mode 100644 index 0e821d6..0000000 --- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/util/ExpressionValidator.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.oodt.cas.pushpull.util; - -//AWT Imports - -import java.awt.BorderLayout; -import java.awt.Color; -import java.awt.Dimension; -import java.awt.GridLayout; -import java.awt.Toolkit; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.swing.BorderFactory; -import javax.swing.JButton; -import javax.swing.JFormattedTextField; -import javax.swing.JFrame; -import javax.swing.JLabel; -import javax.swing.JPanel; -import javax.swing.SwingUtilities; - -//Swing Imports - -/** - * - * @author bfoster - * @version $Revision$ - * - * <p>Use to evaluate whether your regular expressions are - * accepting the correct strings - * </p>. - */ -public class ExpressionValidator extends JPanel { - private static Logger LOG = Logger.getLogger(ExpressionValidator.class.getName()); - private static final long serialVersionUID = -2840589940304298547L; - - private JLabel exprLabel; - private JLabel validateLabel; - private JLabel resultLabel; - - private JFormattedTextField exprField; - private JFormattedTextField validateField; - private JFormattedTextField resultField; - - public ExpressionValidator() { - super(new BorderLayout()); - - exprLabel = new JLabel("Regular Expression: "); - validateLabel = new JLabel("Validate String: "); - resultLabel = new JLabel("Result: "); - - exprField = new JFormattedTextField(); - exprField.setColumns(60); - - validateField = new JFormattedTextField(); - validateField.setColumns(60); - - resultField = new JFormattedTextField(); - resultField.setColumns(60); - resultField.setEditable(false); - resultField.setForeground(Color.red); - - exprLabel.setLabelFor(exprField); - validateLabel.setLabelFor(validateField); - resultLabel.setLabelFor(resultField); - - JPanel labelPane = new JPanel(new GridLayout(0, 1)); - labelPane.add(exprLabel); - labelPane.add(validateLabel); - labelPane.add(resultLabel); - - JPanel fieldPane = new JPanel(new GridLayout(0, 1)); - fieldPane.add(exprField); - fieldPane.add(validateField); - fieldPane.add(resultField); - - JPanel buttonPane = new JPanel(new BorderLayout()); - JButton validate = new JButton("Validate"); - validate.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent event) { - try { - boolean value = ((String) validateField.getText()).matches((String) exprField.getText()); - resultField.setText(value + ""); - }catch (Exception e) { - LOG.log(Level.SEVERE, e.getMessage()); - } - } - }); - buttonPane.add(validate, BorderLayout.EAST); - - setBorder(BorderFactory.createEmptyBorder(20, 20, 20, 20)); - add(labelPane, BorderLayout.WEST); - add(fieldPane, BorderLayout.CENTER); - add(buttonPane, BorderLayout.EAST); - } - - public static void main(String[] args) { - SwingUtilities.invokeLater(new Runnable() { - public void run() { - Dimension dim = Toolkit.getDefaultToolkit().getScreenSize(); - JFrame frame = new JFrame("Regular Expression Evaluator"); - frame.setSize(dim.width / 2, dim.height / 2); - frame.setLocation(dim.width / 3, dim.height / 3); - frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); - frame.add(new ExpressionValidator()); - frame.pack(); - frame.setVisible(true); - } - }); - } - -}
http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/REMOVE.log ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/REMOVE.log b/pushpull/src/main/resources/REMOVE.log deleted file mode 100644 index cb27d16..0000000 --- a/pushpull/src/main/resources/REMOVE.log +++ /dev/null @@ -1,18 +0,0 @@ -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -You can remove this file. It was only included to ensure that the log directory for this -distribution was created on assembly. http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/default.properties ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/default.properties b/pushpull/src/main/resources/default.properties deleted file mode 100644 index a50664d..0000000 --- a/pushpull/src/main/resources/default.properties +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE.txt file distributed with -# this work for additional information regarding copyright ownership. The ASF -# licenses this file to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -#-------------------------------------------------------# -#-----------------Protocol Defaults---------------------# -#-------------------------------------------------------# - -#configuration to make java.net.URL accept unsupported protocols -java.protocol.handler.pkgs=org.apache.oodt.cas.pushpull.url.handlers http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/documentation/Push-Pull-User-Manual_2008-02-12.doc ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/documentation/Push-Pull-User-Manual_2008-02-12.doc b/pushpull/src/main/resources/documentation/Push-Pull-User-Manual_2008-02-12.doc deleted file mode 100755 index 5fb5e28..0000000 Binary files a/pushpull/src/main/resources/documentation/Push-Pull-User-Manual_2008-02-12.doc and /dev/null differ http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/ClassNoaaEmailParserFiles/IasiEmail ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/ClassNoaaEmailParserFiles/IasiEmail b/pushpull/src/main/resources/examples/ClassNoaaEmailParserFiles/IasiEmail deleted file mode 100644 index 7141348..0000000 --- a/pushpull/src/main/resources/examples/ClassNoaaEmailParserFiles/IasiEmail +++ /dev/null @@ -1,129 +0,0 @@ - - - NOAA COMPREHENSIVE LARGE ARRAY-DATA STEWARDSHIP SYSTEM - - A SERVICE OF: - NOAA NATIONAL DATA CENTERS - - -The CLASS has processed your order number A1040663. The total size of -this order is 1246002618 bytes. - - -Items are available for you in the anonymous FTP area. -You may obtain them as follows: - - ftp ftp.class.noaa.gov - Logon to CLASS system - anonymous - login userid - user@internet - enter your internet address as a password - binary - changes transfer mode to binary - cd A1040663 - changes to correct directory - get -IASI_xxx_1C_M02_20070803000257Z_20070803000553Z_N_O_20070803013826Z__20070803014 -346 - - copies data; size is 60242714 bytes - get -IASI_xxx_1C_M02_20070802235953Z_20070803000257Z_N_O_20070803013555Z__20070803014 -041 - - copies data; size is 62970482 bytes - get -IASI_xxx_1C_M02_20070803000553Z_20070803000857Z_N_O_20070803014058Z__20070803014 -616 - - copies data; size is 62970482 bytes - get -IASI_xxx_1C_M02_20070803000857Z_20070803000953Z_N_O_20070803014332Z__20070803014 -646 - - copies data; size is 19326194 bytes - get -IASI_xxx_1C_M02_20070803001218Z_20070803001457Z_N_O_20070803014438Z__20070803014 -957 - - copies data; size is 54787178 bytes - get -IASI_xxx_1C_M02_20070803001458Z_20070803001753Z_N_O_20070803014619Z__20070803015 -222 - - copies data; size is 60242714 bytes - get -IASI_xxx_1C_M02_20070803001754Z_20070803002057Z_N_O_20070803014825Z__20070803015 -507 - - copies data; size is 62970482 bytes - get -IASI_xxx_1C_M02_20070803002058Z_20070803002353Z_N_O_20070803015110Z__20070803015 -742 - - copies data; size is 60242714 bytes - get -IASI_xxx_1C_M02_20070803002354Z_20070803002657Z_N_O_20070803015337Z__20070803020 -042 - - copies data; size is 62970482 bytes - get -IASI_xxx_1C_M02_20070803002658Z_20070803002953Z_N_O_20070803015605Z__20070803020 -202 - - copies data; size is 60242714 bytes - get -IASI_xxx_1C_M02_20070803002954Z_20070803003257Z_N_O_20070803021709Z__20070803022 -207 - - copies data; size is 62970482 bytes - get -IASI_xxx_1C_M02_20070803004458Z_20070803004753Z_N_O_20070803022947Z__20070803023 -437 - - copies data; size is 60242714 bytes - get -IASI_xxx_1C_M02_20070803003554Z_20070803003857Z_N_O_20070803022252Z__20070803022 -802 - - copies data; size is 62970482 bytes - get -IASI_xxx_1C_M02_20070803003858Z_20070803004153Z_N_O_20070803022515Z__20070803023 -102 - - copies data; size is 60242714 bytes - get -IASI_xxx_1C_M02_20070803004154Z_20070803004457Z_N_O_20070803022746Z__20070803023 -352 - - copies data; size is 62970482 bytes - get -IASI_xxx_1C_M02_20070803003258Z_20070803003553Z_N_O_20070803021943Z__20070803022 -507 - - copies data; size is 60242714 bytes - get -IASI_xxx_1C_M02_20070803005058Z_20070803005353Z_N_O_20070803023436Z__20070803024 -017 - - copies data; size is 60242714 bytes - get -IASI_xxx_1C_M02_20070803005354Z_20070803005657Z_N_O_20070803023700Z__20070803024 -132 - - copies data; size is 62970482 bytes - get -IASI_xxx_1C_M02_20070803005658Z_20070803005953Z_N_O_20070803023936Z__20070803024 -357 - - copies data; size is 60242714 bytes - get -IASI_xxx_1C_M02_20070803005954Z_20070803010257Z_N_O_20070803024159Z__20070803024 -657 - - copies data; size is 62970482 bytes - get -IASI_xxx_1C_M02_20070803004754Z_20070803005057Z_N_O_20070803023212Z__20070803023 -737 - - copies data; size is 62970482 bytes - bye (or quit) - to end ftp session. - -Alternatively, you can also pick up your data set via -ftp://ftp.class.noaa.gov/A1040663/ - or -http://www.class.noaa.gov/download/A1040663 - -NOTE: You must pick up your data within 120 hours of this notice. - - - - -Thank you. - - -<< DO NOT reply to this e-mail as it is automatically generated. >> -<< Please direct questions to the Help Desk: >> - - - NOAA/CLASS User Assistance - National Climatic Data Center - 151 Patton Avenue - Asheville, NC 28801-5001 - - Internet: [email protected] http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRABRAD.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRABRAD.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRABRAD.xml deleted file mode 100644 index d7b0491..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRABRAD.xml +++ /dev/null @@ -1,26 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="/ftp/data/s4pa/Aqua_AIRS_Level1/AIRABRAD.005/2006"> - <nofiles/> - <dir name="\d{3}"> - <nodirs/> - <file name="AIRS\.2006\.\d{2}\.\d{2}\.\d{3}.L1B.AMSU_Rad\.v5\.0\.0\.0\.G\d{11}\.hdf(\.xml){0,1}"/> - </dir> - </dirstruct> -</root> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRS3ST8.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRS3ST8.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRS3ST8.xml deleted file mode 100644 index bae8cea..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRS3ST8.xml +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="/data/s4pa/Aqua_AIRS_Level3/AIRS3ST8.005/2007"> - <nodirs/> - <file name="AIRS\.2007.\d{2}.\d{2}.L3.RetStd_IR008\.v5\.0\.14\.0\.G\d{11}\.hdf(\.xml){0,1}"/> - </dirstruct> -</root> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRS3STD.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRS3STD.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRS3STD.xml deleted file mode 100644 index 8b45a34..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/AIRS3STD.xml +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="/data/s4pa/Aqua_AIRS_Level3/AIRS3STD.005/2007"> - <nodirs/> - <file name="AIRS\.2007.\d{2}.\d{2}.L3.RetStd_IR001\.v5\.0\.14\.0\.G\d{11}\.hdf(\.xml){0,1}"/> - </dirstruct> -</root> http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CLASS_Email.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CLASS_Email.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CLASS_Email.xml deleted file mode 100644 index 0c55af3..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CLASS_Email.xml +++ /dev/null @@ -1,22 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="INBOX"> - <nodirs/> - </dirstruct> -</root> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CLASS_Text_Email.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CLASS_Text_Email.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CLASS_Text_Email.xml deleted file mode 100644 index c4399c5..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CLASS_Text_Email.xml +++ /dev/null @@ -1,26 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="/home/class/TextMail"> - <dir name="\d{4}-\d{2}-\d{2}_\d{2}"> - <dir name="class"> - <nodirs/> - </dir> - </dir> - </dirstruct> -</root> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CloudSAT.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CloudSAT.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CloudSAT.xml deleted file mode 100644 index c3477ae..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/CloudSAT.xml +++ /dev/null @@ -1,20 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="2B-CWC-RO.R04/2007/304"/> -</root> http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/GFS.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/GFS.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/GFS.xml deleted file mode 100644 index 6ceea12..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/GFS.xml +++ /dev/null @@ -1,30 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="/pub/data/nccf/com/gfs/prod"> - <nofiles/> - <dir name="gfs\.\d{8}((12)|(00))"> - <nodirs/> - <file name="gfs\.t((00)|(12))z\.pgrb2f((03)|(06)|(09)|(12))"/> - </dir> - </dirstruct> -</root> - - - - \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IASI.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IASI.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IASI.xml deleted file mode 100644 index d63e871..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IASI.xml +++ /dev/null @@ -1,22 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="/misc/raid2/IASI/L1C/2007"> - <nofiles/> - </dirstruct> -</root> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IASI_2.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IASI_2.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IASI_2.xml deleted file mode 100644 index 30bdbd8..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IASI_2.xml +++ /dev/null @@ -1,22 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct root_based="true" starting_path="/A6719183"> - <nodirs/> - </dirstruct> -</root> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IBIBLIO.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IBIBLIO.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IBIBLIO.xml deleted file mode 100644 index 009e297..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/IBIBLIO.xml +++ /dev/null @@ -1,25 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="/mirrors/apache/"> - <nofiles/> - <dir name="ant"> - <nodirs/> - </dir> - </dirstruct> -</root> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/MODIS.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/MODIS.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/MODIS.xml deleted file mode 100644 index 12cf2d9..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/MODIS.xml +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="/MOTA/MCD43B1.005/2007.02.02"> - <nodirs/> - <file name="MCD43B1\.A2007033\.h\d{2}v\d{2}\.005\.\d{13}\.hdf(\.xml){0,1}"/> - </dirstruct> -</root> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/DirStructXmlParserFiles/RTG_SST.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/RTG_SST.xml b/pushpull/src/main/resources/examples/DirStructXmlParserFiles/RTG_SST.xml deleted file mode 100644 index 0dfbc5f..0000000 --- a/pushpull/src/main/resources/examples/DirStructXmlParserFiles/RTG_SST.xml +++ /dev/null @@ -1,24 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<root> - <dirstruct starting_path="/pub/history/sst"> - <nodirs/> - <file name="rtg_sst_grb_0\.5\.\d{8}"/> - </dirstruct> -</root> - http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/ExternalSourcesFiles/ExternalSources.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/ExternalSourcesFiles/ExternalSources.xml b/pushpull/src/main/resources/examples/ExternalSourcesFiles/ExternalSources.xml deleted file mode 100644 index 242b883..0000000 --- a/pushpull/src/main/resources/examples/ExternalSourcesFiles/ExternalSources.xml +++ /dev/null @@ -1,75 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<sources> - <source host="polar.ncep.noaa.gov"> - <login type="ftp" alias="PolarNcepNoaaFtp"> - <username>anonymous</username> - <password>[email protected]</password> - </login> - </source> - <source host="ftp1.cloudsat.cira.colostate.edu"> - <login type="ftp" alias="CloudSATFtp"> - <username></username> - <password></password> - <cdTestDir>2B-CWC-RO.R04/2007/304</cdTestDir> - <maxConn>5</maxConn> - </login> - </source> - <source host="ftp.class.noaa.gov"> - <login type="ftp" alias="ClassFtp"> - <username>anonymous</username> - <password>[email protected]</password> - </login> - </source> - <source host="localhost"> - <login type="file" alias="localhost"> - <username>none</username> - <password>none</password> - </login> - </source> - <source host="ftpprd.ncep.noaa.gov"> - <login type="ftp" alias="GfsFtp"> - <username>anonymous</username> - <password>[email protected]</password> - </login> - </source> - <source host="e4ftl01u.ecs.nasa.gov"> - <login type="ftp" alias="ModisFtp"> - <username>anonymous</username> - <password>[email protected]</password> - </login> - </source> - <source host="airscal2u.ecs.nasa.gov"> - <login type="ftp" alias="GdaacFtp"> - <username>anonymous</username> - <password>[email protected]</password> - </login> - </source> - <source host="acdisc.gsfc.nasa.gov"> - <login type="ftp" alias="AcdiscGsfcFtp"> - <username>anonymous</username> - <password>[email protected]</password> - </login> - </source> - <source host="www.signal42.com"> - <login type="http" alias="Signal42ApacheMirrorHttp"> - <username>none</username> - <password>none</password> - </login> - </source> -</sources> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/FileListParserFiles/DownloadList.txt ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/FileListParserFiles/DownloadList.txt b/pushpull/src/main/resources/examples/FileListParserFiles/DownloadList.txt deleted file mode 100644 index 9280931..0000000 --- a/pushpull/src/main/resources/examples/FileListParserFiles/DownloadList.txt +++ /dev/null @@ -1,7 +0,0 @@ -/misc/raid2/IASI/L1C/2007/06/30 -IASI_xxx_1C_M02_20070629235953Z_20070630000256Z_N_O_20070630013723Z__20070630014330 -IASI_xxx_1C_M02_20070630000257Z_20070630000552Z_N_O_20070630013952Z__20070630014640 -IASI_xxx_1C_M02_20070630000553Z_20070630000856Z_N_O_20070630014220Z__20070630014740 -IASI_xxx_1C_M02_20070630000857Z_20070630001152Z_N_O_20070630014452Z__20070630014944 -IASI_xxx_1C_M02_20070630001153Z_20070630001456Z_N_O_20070630014718Z__20070630015234 -IASI_xxx_1C_M02_20070630001753Z_20070630002056Z_N_O_20070630015228Z__20070630015909 http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]forward ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]forward b/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]forward deleted file mode 100644 index 4f0d5bc..0000000 --- a/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]forward +++ /dev/null @@ -1 +0,0 @@ -"|IFS=' '&&p=/usr/local/bin/procmail&&test -f $p&&exec $p -f-||exit 75#class" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]procmail/general.rc ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]procmail/general.rc b/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]procmail/general.rc deleted file mode 100644 index 901cfd7..0000000 --- a/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]procmail/general.rc +++ /dev/null @@ -1,25 +0,0 @@ -MONTHFOLDER= `date +%Y-%m-%d_%H ` -DUMMY= `test -d $MONTHFOLDER || mkdir -m g=rwx $MONTHFOLDER ` - -CLASS_FOLDER= ${MONTHFOLDER}/class -UNKNOWN_FOLDER= ${MONTHFOLDER}/unknown -JPL_FOLDER= ${MONTHFOLDER}/jpl -DUMMY= `test -d $CLASS_FOLDER || mkdir -m g=rwx $CLASS_FOLDER ` -DUMMY= `test -d $UNKNOWN_FOLDER || mkdir -m g=rwx $UNKNOWN_FOLDER ` -DUMMY= `test -d $JPL_FOLDER || mkdir -m g=rwx $JPL_FOLDER ` - -:0 fw -* -| formail -k -X From: -X Subject: - -:0 : -* ^From.*class\.noaa\.gov -${CLASS_FOLDER} - -:0 : -* ^From.*jpl\.nasa\.gov -${JPL_FOLDER} - -:0 : -* -${UNKNOWN_FOLDER} http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]procmailrc ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]procmailrc b/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]procmailrc deleted file mode 100644 index 65524a1..0000000 --- a/pushpull/src/main/resources/examples/ProcmailForwarding/[dot]procmailrc +++ /dev/null @@ -1,7 +0,0 @@ -VERBOSE=off -UMASK=007 -MAILDIR=$HOME/TextMail -PMDIR=$HOME/.procmail -DEFAULT=$MAILDIR -LOGFILE=$PMDIR/log -INCLUDERC=$PMDIR/general.rc http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/examples/RemoteSpecsFiles/RemoteSpecs.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/examples/RemoteSpecsFiles/RemoteSpecs.xml b/pushpull/src/main/resources/examples/RemoteSpecsFiles/RemoteSpecs.xml deleted file mode 100644 index bf4ed22..0000000 --- a/pushpull/src/main/resources/examples/RemoteSpecsFiles/RemoteSpecs.xml +++ /dev/null @@ -1,181 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<remoteSpecs> - - <aliasSpecs> - <aliasSpec file="[CAS_PP_RESOURCES]/examples/ExternalSourcesFiles/ExternalSources.xml"/> - </aliasSpecs> - - <daemons> - - <daemon alias="DewFtp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="IASI\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="DewFtp" deleteFromServer="no"/> - </daemon> - - <daemon alias="DewFtp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_TEMP_PROPS]/WeatherSftp"> - <downloadInfo alias="WeatherSftp" remoteDir="junkTest"/> - <propFiles regExp="IASI\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="DewFtp" deleteFromServer="no"/> - </daemon> - - <daemon alias="CalPolySftp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="CalPoly\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="CalPolySftp" deleteFromServer="no"/> - </daemon> - - <daemon alias="WeatherSftp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="IASI\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="WeatherSftp" deleteFromServer="no"/> - </daemon> - - <daemon alias="WeatherSftp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_TEMP_PROPS]/WeatherSftp"> - <downloadInfo alias="WeatherSftp" remoteDir="junkTest"/> - <propFiles regExp="IASI\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="WeatherSftp" deleteFromServer="no" queryElement="RetrievedFromLoc"/> - </daemon> - - <daemon alias="WeatherSftp" active="no"> - <runInfo firstRunDateTime="2007-12-24T00:10:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/FileListParserFiles"> - <propFiles regExp="DownloadList\.txt" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.FileListParser"/> - </propInfo> - <dataInfo stagingArea="WeatherSftp" deleteFromServer="no" queryElement="RetrievedFromLoc"/> - </daemon> - - <daemon alias="ClassFtp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="IASI_2.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="ClassFtp" deleteFromServer="no" queryElement="Filename"/> - </daemon> - - <daemon alias="ClassFtp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/ClassNoaaEmailParserFiles"> - <propFiles regExp="[^\.].*" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.ClassNoaaEmailParser"/> - </propInfo> - <dataInfo stagingArea="ClassFtp" deleteFromServer="no" queryElement="Filename"/> - </daemon> - - <daemon alias="ClassFtp" active="no"> - <runInfo firstRunDateTime="2007-12-14T00:00:00Z" period="3m" epsilon="20s" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_TEMP_PROPS]/ClassFtp/Emails"> - <downloadInfo alias="PeateImaps" deleteFromServer="yes" renamingConv="CLASS_[HOST]_[DATE.UTC]_[FILENAME].email"> - <propFile path="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles/CLASS_Email.xml" - parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </downloadInfo> - <propFiles regExp=".*\.email" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.ClassNoaaEmailParser"/> - <afterUse moveToOnSuccess="[CAS_PP_TEMP_PROPS]/ClassFtp/ProcessedEmails" - moveToOnFail="[CAS_PP_TEMP_PROPS]/ClassFtp/FailedEmails"/> - </propInfo> - <dataInfo stagingArea="ClassFtp" deleteFromServer="no" allowAliasOverride="yes" - queryElement="Filename" renamingConv="[GREP_RM('^L\d{1,}?\.','[FILENAME]')]"/> - </daemon> - - <daemon alias="ClassFtp" active="yes"> - <runInfo firstRunDateTime="2007-12-14T00:00:00Z" period="3m" epsilon="20s" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_TEMP_PROPS]/ClassFtp/Emails"> - <downloadInfo alias="localhost" deleteFromServer="yes" renamingConv="CLASS_[HOST]_[DATE.UTC]_[FILENAME].email"> - <propFile path="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles/CLASS_Text_Email.xml" - parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </downloadInfo> - <propFiles regExp=".*\.email" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.ClassNoaaEmailParser"/> - <afterUse moveToOnSuccess="[CAS_PP_TEMP_PROPS]/ClassFtp/ProcessedEmails" - moveToOnFail="[CAS_PP_TEMP_PROPS]/ClassFtp/FailedEmails"/> - </propInfo> - <dataInfo stagingArea="ClassFtp" deleteFromServer="no" allowAliasOverride="yes" - queryElement="Filename"/> - </daemon> - - <daemon alias="GfsFtp" active="no"> - <runInfo firstRunDateTime="2007-11-27T00:00:00Z" period="3m" epsilon="20s" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="GFS\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="GfsFtp" renamingConv="[PARENT_FILENAME]_[FILENAME]" queryElement="RetrievedFromLoc"/> - </daemon> - - <daemon alias="PolarNcepNoaaFtp" active="no"> - <runInfo firstRunDateTime="2007-11-27T00:00:00Z" period="3m" epsilon="20s" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="RTG_SST\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="RtgSstFtp" queryElement="Filename"/> - </daemon> - - <daemon alias="ModisFtp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="MODIS\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="ModisFtp" deleteFromServer="no" queryElement="RetrievedFromLoc"/> - </daemon> - - <daemon alias="GdaacFtp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="AIRABRAD\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="GdaccFtp" deleteFromServer="no" queryElement="RetrievedFromLoc"/> - </daemon> - - <daemon alias="AcdiscGsfcFtp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="AIRS3STD\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - <propFiles regExp="AIRS3ST8\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="AcdiscGsfcFtp" deleteFromServer="no" queryElement="RetrievedFromLoc"/> - </daemon> - - <daemon alias="Signal42ApacheMirrorHttp" active="no"> - <runInfo firstRunDateTime="2007-12-01T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="IBIBLIO\.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="Signal42ApacheMirrorHttp" deleteFromServer="no" queryElement="RetrievedFromLoc"/> - </daemon> - - <daemon alias="RimeSftp" active="no"> - <runInfo firstRunDateTime="2008-02-19T00:00:00Z" period="3m" runOnReboot="yes"/> - <propInfo dir="[CAS_PP_RESOURCES]/examples/DirStructXmlParserFiles"> - <propFiles regExp="RimeDIRSTRUCT.xml" parser="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </propInfo> - <dataInfo stagingArea="RimeSftp" renamingConv="[PATH_NO_FILENAME]/[FILENAME]" deleteFromServer="no" queryElement="RetrievedFromLoc"/> - </daemon> - - </daemons> - -</remoteSpecs> http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/jssecacerts ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/jssecacerts b/pushpull/src/main/resources/jssecacerts deleted file mode 100644 index 7c8f621..0000000 Binary files a/pushpull/src/main/resources/jssecacerts and /dev/null differ http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/logging.properties ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/logging.properties b/pushpull/src/main/resources/logging.properties deleted file mode 100644 index 6af123f..0000000 --- a/pushpull/src/main/resources/logging.properties +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE.txt file distributed with -# this work for additional information regarding copyright ownership. The ASF -# licenses this file to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -# Specify the handlers to create in the root logger -# (all loggers are children of the root logger) -# The following creates two handlers -handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler - -# Set the default logging level for the root logger -.level = ALL - -# Set the default logging level for new ConsoleHandler instances -java.util.logging.ConsoleHandler.level = ALL -java.util.logging.FileHandler.level = ALL - -# Set the default formatter for new ConsoleHandler instances -java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter - -# default file output is in user's home directory. -java.util.logging.FileHandler.pattern = ../logs/cas-pushpull%g.log -java.util.logging.FileHandler.limit = 50000 -java.util.logging.FileHandler.count = 5 -java.util.logging.FileHandler.append = true -java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter - -java.lang.management.level = OFF -java.rmi.registry.level = OFF -javax.management.level = OFF -javax.management.level = OFF -sun.rmi.level = OFF http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/policy/ParserToRetrievalMethodMap.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/policy/ParserToRetrievalMethodMap.xml b/pushpull/src/main/resources/policy/ParserToRetrievalMethodMap.xml deleted file mode 100644 index 537624c..0000000 --- a/pushpull/src/main/resources/policy/ParserToRetrievalMethodMap.xml +++ /dev/null @@ -1,30 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<rtvlMethods> - - <rtvlMethod class="org.apache.oodt.cas.pushpull.retrievalmethod.RemoteCrawler"> - <parser class="org.apache.oodt.cas.pushpull.filerestrictions.parsers.DirStructXmlParser"/> - </rtvlMethod> - - <rtvlMethod class="org.apache.oodt.cas.pushpull.retrievalmethod.ListRetriever"> - <parser class="org.apache.oodt.cas.pushpull.filerestrictions.parsers.FileListParser"/> - <parser class="org.apache.oodt.cas.pushpull.filerestrictions.parsers.ClassNoaaEmailParser"/> - <parser class="org.apache.oodt.cas.pushpull.filerestrictions.parsers.GenericEmailParser"/> - </rtvlMethod> - -</rtvlMethods> http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/policy/ProtocolFactoryInfo.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/policy/ProtocolFactoryInfo.xml b/pushpull/src/main/resources/policy/ProtocolFactoryInfo.xml deleted file mode 100644 index d3c8c4e..0000000 --- a/pushpull/src/main/resources/policy/ProtocolFactoryInfo.xml +++ /dev/null @@ -1,45 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<protocols> - - <protocol type="ftp"> - <protocolFactory class="org.apache.oodt.cas.protocol.ftp.CogJGlobusFtpProtocolFactory"/> - <protocolFactory class="org.apache.oodt.cas.protocol.ftp.CommonsNetFtpProtocolFactory"/> - </protocol> - - <protocol type="http"> - <protocolFactory class="org.apache.oodt.cas.protocol.http.HttpProtocolFactory"/> - </protocol> - - <protocol type="sftp"> - <protocolFactory class="org.apache.oodt.cas.protocol.sftp.JschSftpProtocolFactory"/> - </protocol> - - <protocol type="imaps"> - <protocolFactory class="org.apache.oodt.cas.protocol.imaps.ImapsProtocolFactory"/> - </protocol> - - <!-- - Note: hasn't been implemented yet, see OODT-537 - - <protocol type="file"> - <protocolFactory class="org.apache.oodt.cas.protocol.local.LocalProtocolFactory"/> - </protocol> - --> - -</protocols> http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/policy/mimetypes.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/policy/mimetypes.xml b/pushpull/src/main/resources/policy/mimetypes.xml deleted file mode 100644 index 36a5b98..0000000 --- a/pushpull/src/main/resources/policy/mimetypes.xml +++ /dev/null @@ -1,60 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<mime-info> - - <mime-type type="product/pgp_signature"> - <_comment><![CDATA[ProductNameGenerator=[GREP_RM('\.sig$','[FILENAME]')]]]></_comment> - <glob pattern="*.sig"/> - </mime-type> - - <mime-type type="metadata/cas_pushpull"> - <glob pattern="*.info.tmp"/> - </mime-type> - - <mime-type type="metadata/cas_metadata"> - <glob pattern="*.cas"/> - <glob pattern="*.met"/> - </mime-type> - - <mime-type type="product/iasi_xxx_1c"> - <sub-class-of type="metop_a/noaa"/> - <_comment>ProductType=MOA_IASI_L1C</_comment> - <glob pattern="(?:L\d{1,}\.){0,1}IASI_xxx_1C_\w{3}_\w{15}_\w{15}_\w_\w_\w{15}__\w{14}" isregex="true"/> - </mime-type> - - <mime-type type="product/mhsx"> - <sub-class-of type="metop_a/noaa"/> - <_comment>ProductType=MOA_MHS_L1B</_comment> - <glob pattern="(?:L\d{1,}\.){0,1}NSS\.MHSX\.\w{2}\.D\d{5}\.S\d{4}\.E\d{4}\.B\d{7}\.\w{2}" isregex="true"/> - </mime-type> - - <mime-type type="product/amsu_a"> - <sub-class-of type="metop_a/noaa"/> - <_comment>ProductType=MOA_AMSUA_L1B</_comment> - <glob pattern="(?:L\d{1,}\.){0,1}NSS\.AMAX\.\w{2}\.D\d{5}\.S\d{4}\.E\d{4}\.B\d{7}\.\w{2}" isregex="true"/> - </mime-type> - - <mime-type type="product/rtg_sst"> - <sub-class-of type="ncep/model"/> - <_comment><![CDATA[ProductType=RTG_SST && UniqueElement=RetrievedFromLoc]]></_comment> - <glob pattern="rtg_sst_grb_0\.5\.\d{8}" isregex="true"/> - <glob pattern="sst\.\d{8}_rtgssthr_grb_0\.5\.grib2" isregex="true"/> - </mime-type> - -</mime-info> - http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/policy/mimetypes.xsd ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/policy/mimetypes.xsd b/pushpull/src/main/resources/policy/mimetypes.xsd deleted file mode 100644 index 4b795e2..0000000 --- a/pushpull/src/main/resources/policy/mimetypes.xsd +++ /dev/null @@ -1,71 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"; - xmlns="http://oodt.apache.org/schema/pushpull/mimetypes.xsd"; - targetNamespace="http://oodt.apache.org/schema/pushpull/mimetypes.xsd"; - elementFormDefault="qualified" vc:minVersion="1.1" - xmlns:vc="http://www.w3.org/2007/XMLSchema-versioning";> - <xs:annotation> - <xs:documentation> - This schema defines how mimetypes.xml file entries should be - structured within the context of the OODT pushpull component. - </xs:documentation> - </xs:annotation> - <xs:element name="mime-info"> - <xs:annotation> - <xs:documentation> - mime-info is the root element containing one or more 'mime-type'. - </xs:documentation> - </xs:annotation> - <xs:complexType> - <xs:sequence> - <xs:element maxOccurs="unbounded" name="mime-type"> - <xs:annotation> - <xs:documentation> - There can be an unbounded number of 'mime-type' elements each containing - zero or more 'sub-class-of' and 'glob' children but only one 'comment' - child. - </xs:documentation> - </xs:annotation> - <xs:complexType> - <xs:all> - <xs:element maxOccurs="unbounded" name="sub-class-of"> - <xs:complexType> - <xs:attribute name="type" type="xs:string"/> - </xs:complexType> - </xs:element> - <xs:element name="_comment"/> - <xs:element maxOccurs="unbounded" name="glob"> - <xs:annotation> - <xs:documentation> - - </xs:documentation> - </xs:annotation> - <xs:complexType> - <xs:attribute name="pattern" type="xs:string"/> - <xs:attribute name="isRegex" type="xs:boolean"/> - </xs:complexType> - </xs:element> - </xs:all> - <xs:attribute name="type" type="xs:string"/> - </xs:complexType> - </xs:element> - </xs:sequence> - </xs:complexType> - </xs:element> -</xs:schema> http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/resources/push_pull_framework.properties ---------------------------------------------------------------------- diff --git a/pushpull/src/main/resources/push_pull_framework.properties b/pushpull/src/main/resources/push_pull_framework.properties deleted file mode 100644 index d1726ba..0000000 --- a/pushpull/src/main/resources/push_pull_framework.properties +++ /dev/null @@ -1,106 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE.txt file distributed with -# this work for additional information regarding copyright ownership. The ASF -# licenses this file to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# $Id: Configuration properties for the CAS Push-Pull-Framework$ - -#---------------------------------------------------------------# -#-------------------- Default Configurations -------------------# -#---------------------------------------------------------------# - -#external configuration files -org.apache.oodt.cas.pushpull.config.external.properties.files=[CAS_PP_HOME]/etc/default.properties - - -#---------------------------------------------------------------# -#-------------------- Default Ingester Properties --------------# -#---------------------------------------------------------------# - -#ingester classpath -org.apache.oodt.cas.filemgr.ingester=org.apache.oodt.cas.filemgr.ingest.StdIngester - -#ingester transferer -org.apache.oodt.cas.filemgr.datatransfer.factory=org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory - -# ingester filemgr url -org.apache.oodt.cas.filemgr.url=[FILEMGR_LOC] - -#---------------------------------------------------------------# -#-------------------- Cached Ingester Properties --------------# -#---------------------------------------------------------------# -org.apache.oodt.cas.filemgr.ingest.cache.factory= - -org.apache.oodt.cas.filemgr.ingest.cache.rangeQueryElementName= - -org.apache.oodt.cas.filemgr.ingest.cache.range.start= - -org.apache.oodt.cas.filemgr.ingest.cache.range.end= - -org.apache.oodt.cas.filemgr.ingest.cache.uniqueElementName= - -org.apache.oodt.cas.filemgr.ingest.cache.productType= - - -#-------------------------------------------------------------# -#---------------- Protocol Configurations --------------------# -#-------------------------------------------------------------# - -#set the paging size (use -1 to turn paging off) -org.apache.oodt.cas.pushpull.protocol.page_size=8 - -#protocolfactory specification for protocol types -org.apache.oodt.cas.pushpull.config.protocolfactory.info.files=[CAS_PP_HOME]/policy/ProtocolFactoryInfo.xml - - -#-------------------------------------------------------------# -#------------ FileRetrievalSystem Configurations -------------# -#-------------------------------------------------------------# - -#parser to retrievalmethod map -org.apache.oodt.cas.pushpull.config.parser.info.files=[CAS_PP_HOME]/policy/ParserToRetrievalMethodMap.xml - -#unique metadata element info -org.apache.oodt.cas.pushpull.config.type.detection.file=[CAS_PP_HOME]/policy/mimetypes.xml - -#directory below which all data file will be downloaded to -org.apache.oodt.cas.pushpull.data.files.base.staging.area=[CAS_PP_STAGING_AREA] - -#list of metadata values to print to metadata file -org.apache.oodt.cas.pushpull.metadata.list.to.print=ProductName,RetrievedFromLoc,DataProvider,FileSize,RenamingString,ProductType,DownloadToDir,DeleteAfterDownload,SuperType - -#extension for temporary metadata file -org.apache.oodt.cas.pushpull.write.met.file=true -org.apache.oodt.cas.pushpull.met.file.extension=info.tmp - -#tracker varries the number of threads used to download files to find the optimized -# number of threads in order to get the best download time -org.apache.oodt.cas.pushpull.crawler.use.tracker=false - -#the recommended number of threads (will be the static number of -# threads used if the tracker is turned off) -org.apache.oodt.cas.pushpull.file.retrieval.system.recommended.thread.count=30 - -#the FileRetrievalSystem will stop accepting files for download once this number -# of failed downloads is reached -org.apache.oodt.cas.pushpull.file.retrieval.system.max.number.allowed.failed.downloads=10 - -#if true, any file not defined in the file type detection file will not be downloaded -org.apache.oodt.cas.pushpull.allow.only.defined.types=true - -org.apache.oodt.cas.pushpull.filerestrictions.parsers.class.noaa.email.parser.contains.exprs=CLASS\ has\ processed,anonymous - -org.apache.oodt.cas.pushpull.generic.email.parser.file.pattern=Wav File: ([^\\s]+) -org.apache.oodt.cas.pushpull.generic.email.parser.check.for.pattern=Pattern to valid email has expected text -org.apache.oodt.cas.pushpull.generic.email.parser.path.to.root=/some/root/path -org.apache.oodt.cas.pushpull.generic.email.parser.metadata.keys=MasterInContactId -org.apache.oodt.cas.pushpull.generic.email.parser.metadata.MasterInContactId=Master ContactID (\d+?)\n http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/site/resources/images/cas-pushpull.jpg ---------------------------------------------------------------------- diff --git a/pushpull/src/site/resources/images/cas-pushpull.jpg b/pushpull/src/site/resources/images/cas-pushpull.jpg deleted file mode 100644 index 26e4345..0000000 Binary files a/pushpull/src/site/resources/images/cas-pushpull.jpg and /dev/null differ http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/site/resources/images/cas-pushpull.psd ---------------------------------------------------------------------- diff --git a/pushpull/src/site/resources/images/cas-pushpull.psd b/pushpull/src/site/resources/images/cas-pushpull.psd deleted file mode 100644 index 8aedbb3..0000000 Binary files a/pushpull/src/site/resources/images/cas-pushpull.psd and /dev/null differ http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/site/resources/images/pp_extension_points.png ---------------------------------------------------------------------- diff --git a/pushpull/src/site/resources/images/pp_extension_points.png b/pushpull/src/site/resources/images/pp_extension_points.png deleted file mode 100644 index b956b84..0000000 Binary files a/pushpull/src/site/resources/images/pp_extension_points.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/site/resources/images/pp_object_model.png ---------------------------------------------------------------------- diff --git a/pushpull/src/site/resources/images/pp_object_model.png b/pushpull/src/site/resources/images/pp_object_model.png deleted file mode 100644 index fd25cd0..0000000 Binary files a/pushpull/src/site/resources/images/pp_object_model.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/site/site.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/site/site.xml b/pushpull/src/site/site.xml deleted file mode 100644 index 4e5af1f..0000000 --- a/pushpull/src/site/site.xml +++ /dev/null @@ -1,32 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more contributor -license agreements. See the NOTICE.txt file distributed with this work for -additional information regarding copyright ownership. The ASF licenses this -file to you under the Apache License, Version 2.0 (the "License"); you may not -use this file except in compliance with the License. You may obtain a copy of -the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations under -the License. ---> -<project name="cas-pushpull"> - - <body> - <links> - <item name="OODT" href="../oodt-site/"/> - </links> - - <menu ref="reports" inherit="bottom"/> - <menu name="Software Documentation"> - <item name="Developer Guide" href="development/developer.html"/> - <item name="Basic User Guide" href="user/basic.html"/> - <item name="Advanced User Guide" href="user/advanced.html"/> - </menu> - </body> -</project> http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/site/xdoc/development/developer.xml ---------------------------------------------------------------------- diff --git a/pushpull/src/site/xdoc/development/developer.xml b/pushpull/src/site/xdoc/development/developer.xml deleted file mode 100644 index 49f522e..0000000 --- a/pushpull/src/site/xdoc/development/developer.xml +++ /dev/null @@ -1,286 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more contributor -license agreements. See the NOTICE.txt file distributed with this work for -additional information regarding copyright ownership. The ASF licenses this -file to you under the Apache License, Version 2.0 (the "License"); you may not -use this file except in compliance with the License. You may obtain a copy of -the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations under -the License. ---> -<document> - <properties> - <title>CAS Push Pull Framework Developer Guide</title> - <author email="[email protected]">Brian Foster</author> - <author email="[email protected]">Chris Mattmann</author> - </properties> - - <body> - - <section name="Introduction"> - <p> - This is the developer guide for the Apache OODT Catalog and Archive Service (CAS) - Push Pull framework, or Push Pull for short. Primarily, this guide - will explain the Push Pull architecture and interfaces, including its - tailorable extension points. For information on installation, configuration, - and examples, please see our <a href="../user/basic.html">User Guides.</a> - - <p>The remainder of this guide is separated into the following sections:</p> - <ul> - <li><a href="#section1">Project Description</a></li> - <li><a href="#section2">Architecture</a></li> - <li><a href="#section3">Extension Points</a></li> - <li><a href="#section4">Current Extension Point Implementations</a></li> - </ul> - - </p> - </section> - - <a name="section1"/> - <section name="Project Description"> - <p>The Push Pull framework is responsible for downloading remote content (pull), - or accepting the delivery of remote content (push) to a local system staging area - for use by the <a href="../../crawler">CAS Crawler Framework</a> to ingest into - the <a href="../../filemgr">CAS File Manager</a>. The Push Pull framework is extensible - and provides a fully tailorable Java-based API for the acquisition of remote content.</p> - </section> - - <a name="section2"/> - <section name="Architecture"> - - <p>In this section, we will describe the architecture of the Push Pull framework, - including its constituent components, object model, and key capabilities.</p> - - <subsection name="Components"> - - <p>The major components of the Push Pull Framework are the Daemon Launcher, the Daemon, - the Protocol Layer, and the File Retrieval System, to name a few. The relationship between - all of these components are shown in the diagram below: - </p> - - <p><img src="../images/pp_extension_points.png" alt="Push Pull Framework Architecture"/></p> - - <p>The Push Pull Framework provides a Daemon Launcher, responsible for creating new - Daemon instances. Each Daemon has an associated Daemon Configuration, and has the - ability to use a File Retrieval Setup extension point. This class is responsible for - leveraging both a Protocol and a File Retrieval System to obtain ProtocolFiles, based on - a File Restrictions Parser, that yields eventually a VirtualFileStructure (VFS) model. The - VFS defines what files to accept and pull down from a remote site. - </p> - - </subsection> - <subsection name="Object Model"> - <p>The critical objects managed by the Push Pull Framework include:</p> - - <ul> - <li><strong>Protocol</strong> - A pluggable means of obtaining content over - some file acquisition method, e.g., FTP, SCP, HTTP, etc.</li> - - <li><strong>Protocol File</strong> - Metadata information about a remote file, - including its ProtocolPath.</li> - - <li><strong>Protocol Path</strong> - A pointer to a remote Product file's (or files') - location, which can be used to derive metadata and determine where to place the file - in the local staging area built by the Push Pull Framework.</li> - - <li><strong>Remote Site</strong> - Descriptive information about a remote site, including - the username/password combination, as well as a origin directory to start interrogating.</li> - </ul> - - <p>Each Protocol delivers one or more Protocol Files. Each ProtocoFile is associated with a - single RemoteSite, and each ProtocolFile is associated with a single ProtocolPath. These - relationships are shown in the below figure.</p> - - <img src="../images/pp_object_model.png" alt="Push Pull Framework Object Model"/> - </subsection> - - <subsection name="Key Capabilities"> - <p>The Push Pull Framework has been designed with a new of key capabilities in mind. - These capabilities include:</p> - - <p><strong>Flexibility</strong> - ability to plug in different Metadata Extractors, - Data Protocols, Content Types, etc.</p> - - <p><strong>Support Push/Pull</strong> - Support of both "Push" and "Pull" - style data transfers.</p> - - <p><strong>Extensibility</strong> - ability to add new, previously undiscovered Data Protocols, - and "plug" them into the framework.</p> - - <p><strong>Java-based</strong> - Use of Java programming language and development kit for Multi-Platform - deployment (using the Java Virtual Machine).</p> - - <p><strong>Fast Data-transfer</strong> - Support of Parallel File Transfers and Data Downloads.</p> - - <p><strong>Email-based Push</strong> - Support for Email-based Push Data Acceptance using IMAP, SMTP protocols.</p> - - <p><strong>Modeling of remote data sites</strong> - Ability to configure âVirtualâ remote directories (based on Metadata - such as Date/Time) to download files from.</p> - - <p><strong>Integration with other CAS components</strong> - Ability to "plug-in" to the CAS File Management - and CAS Crawl Framework components for Data Ingestion.</p> - </subsection> - </section> - - <a name="section3"/> - <section name="Extension Points"> - <p>We have constructed the Push Pull Framework making use of the <i>factory - method pattern</i> to provide multiple extension points for the Push Pull Framework. An - extension point is an interface within the Push Pull Framework that can have many - implementations. This is particularly useful when it comes to software - component configuration because it allows different implementations of - an existing interface to be selected at deployment time.</p> - - <div class="info">The factory method pattern is a creational pattern common to - object oriented design. Each Push Pull Framework extension point involves the - implementation of two interfaces: an <i>extension factory</i> and an - <i>extension</i> implementation. At run-time, the Push Pull Framework loads a - properties file specifies a factory class to use during extension point - instantiation. For example, the Push Pull Framework may communicate with a - remote FTP site to obtain content, or it may use an IMAPS protocol plugin to - accept email-push notifications of available files. - </div> - - <p>Using extension points, it is fairly simple to support many different types - of what are typically referred to as "plug-in architectures". Each of the core - extension points for the Push Pull Framework is described below:</p> - - <table> - <tr> - <td>Protocol</td> - <td>The Protocol extension point is the heart of the Push Pull framework, - responsible for modeling remote sites, and for obtaining their content via - different Retrieval Methods, using different File Restrictions Parsers. - </td> - </tr> - <tr> - <td>Retrieval Method</td> - <td>The Retrieval Method extension point is responsible for orchestrating - download (pull) and acceptance (push) of remote content. - </td> - </tr> - <tr> - <td>File Restrictions Parser</td> - <td>The File Restrictions Parser extension point is responsible for defining - how to accept or decline files encountered by a Retrieval Method, in essence - modeling remote file and directory structures. - </td> - </tr> - <tr> - <td>System</td> - <td>The extension point that provides the external interface to the - Push Pull Framework services. This includes the Daemon Launcher interface, - as well as the associated Daemon interface, that - is managed by with the Daemon Launcher. - </td> - </tr> - - </table> - </section> - - <a name="section4"/> - <section name="Current Extension Point Implementations"> - - <p>There are at least two implementations of all of the aforementioned - extension points for the Push Pull Framework. Each extension point implementation - is detailed in this section.</p> - - <subsection name="Protocol"> - <ul> - <li><strong>Cog JGlobus FTP.</strong> An implementation of - the Protocol extension point for FTP using - <a href="http://dev.globus.org/wiki/CoG_jglobus";>CoG jglobus</a>. - </li> - - <li><strong>Commons Net FTP.</strong> An implementation of the - of the Protocol extension point for FTP using - <a href="http://commons.apache.org/net/";>Commons Net</a> FTP client. - </li> - - <li><strong>HTTP.</strong> An implementation of - the Protocol extension point using Java's URL class, as well as - <a href="http://tika.apche.org/";>Apache Tika</a>'s HTMLParser.</li> - - <li><strong>IMAPS.</strong> An implementation of the - Protocol extension point using IMAPS javax.mail classes from - <a href="http://geronimo.apache.org";>Apache Geronimo</a> and - HTML parsing from <a href="http://tika.apache.org/";>Apache Tika</a>.</li> - - <li><strong>Local.</strong> An implementation of - the Protocol extension point using Java NIO for local - data acquisition.</li> - - <li><strong>SFTP.</strong> An implementation of the - Protocol extension point using <a href="http://www.jcraft.org";>JCraft</a>'s - <a href="http://www.jcraft.org/jsch";>JSch</a> library.</li> - </ul> - </subsection> - - <subsection name="Retrieval Method"> - <ul> - <li><strong>Remote Crawler.</strong> An implementation of - the Retrieval Method interface that uses an XML based set of - policy files to determine which remote directories and files - to crawl and obtain.</li> - - <li><strong>List Retriever.</strong> An implementation of the - Retrieval Method interface that accepts a list of URLs that point - to content to obtain.</li> - </ul> - </subsection> - - <subsection name="File Restrictions Parser"> - <ul> - <li><strong>DirStructXml Parser.</strong> An implementation of - the File Restrictions Parser interface that interprets an XML file - specifying the remote directories and files to obtain.</li> - - <li><strong>FileList Parser.</strong> An implementation of the - File Restrictions Parser interface that specifies an ASCII newline separated - list of URLs pointing to remote directories and files to obtain.</li> - - <li><strong>Class NOAA Email Parser.</strong> An implementation of the - File Restrictions Parser interface that reads email files from NOAA's CLASS - archive which specify lists of directory and file URLs to obtain.</li> - </ul> - </subsection> - - - <subsection name="Daemon Launcher (Daemon client and Daemon server)"> - <ul> - <li><strong>Java RMI based server.</strong> An - implementation of the external server interface for the Push Pull Framework that - uses RMI as the transportation medium to launch Push Pull Daemons.</li> - - <li><strong>Push Pull Daemon.</strong> An - implementation of the client interface for the Java RMI-based - server that uses RMI as the transportation medium to manage and control the - Push Pull services.</li> - </ul> - </subsection> - - </section> - - <section name="Conclusion"> - <p>The aim of this document is to provide information relevant to developers - about the CAS Push Pull Framework. Specifically, this document has described the - Push Pull Framework's architecture, including its constituent components, object model and - key capabilities. Additionally, the this document provides an overview of the - current implementations of the Push Pull Framework's extension points.</p> - - <p>In the <a href="../user/basic.html">Basic User Guide</a> and - <a href="../user/advanced.html">Advanced User Guide</a>, we will cover topics - like installation, configuration, and example uses as well as advanced topics - like scaling and other tips and tricks.</p> - - </section> - - </body> -</document>
