adding datacat system
Project: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/commit/4231ac35 Tree: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/tree/4231ac35 Diff: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/diff/4231ac35 Branch: refs/heads/master Commit: 4231ac353d5fa5cf25eb434f9fbda1052ff15436 Parents: 5425403 Author: Supun Nakandala <[email protected]> Authored: Fri Aug 21 17:07:48 2015 +0530 Committer: Supun Nakandala <[email protected]> Committed: Fri Aug 21 17:07:48 2015 +0530 ---------------------------------------------------------------------- .gitignore | 24 + datacat/.gitignore | 22 + datacat/.gitignore.swp | Bin 0 -> 12288 bytes datacat/LICENSE | 201 + datacat/README.md | 23 + datacat/agent/pom.xml | 132 + .../agent/src/main/assembly/bin-assembly.xml | 77 + .../airavata/datacat/agent/DataCatAgent.java | 84 + .../agent/dispatcher/DispatcherService.java | 66 + .../agent/dispatcher/MetadataDispatcher.java | 178 + .../dispatcher/MetadataDispatcherQueue.java | 84 + .../agent/dispatcher/MonitorDispatcher.java | 133 + .../dispatcher/MonitorDispatcherQueue.java | 84 + .../validators/IMetadataValidator.java | 27 + .../messageBroker/AiravataUpdateListener.java | 148 + .../agent/messageBroker/IMessageBroker.java | 28 + .../agent/messageBroker/RabbitMQProducer.java | 196 + .../agent/messageBroker/RabbitMQPublisher.java | 81 + .../datacat/agent/monitor/IMonitor.java | 37 + .../monitor/impl/LocalFileSystemMonitor.java | 261 + .../datacat/agent/monitor/impl/OutputInfo.java | 62 + .../datacat/agent/util/AgentProperties.java | 75 + .../airavata/datacat/agent/util/Constants.java | 56 + .../datacat/agent/util/ThriftUtils.java | 37 + .../src/main/resources/bin/datacat-agent.sh | 85 + datacat/agent/src/main/resources/bin/setenv.sh | 65 + .../src/main/resources/conf/agent.properties | 44 + .../agent/src/main/resources/log4j.properties | 16 + .../resources/security/client-truststore.jks | Bin 0 -> 35240 bytes .../src/main/resources/security/keystore.jks | Bin 0 -> 33260 bytes .../messageBroker/RabbitMQConsumerTest.java | 124 + .../messageBroker/RabbitMQPublisherTest.java | 78 + datacat/cup_flex/B3PW91.cup | 166 + datacat/cup_flex/B3PW91.flex | 204 + datacat/cup_flex/Makefile | 60 + datacat/cup_flex/casscf.flex | 237 + datacat/cup_flex/cbsQ.cup | 166 + datacat/cup_flex/cbsQ.flex | 204 + datacat/cup_flex/ci.flex | 134 + datacat/cup_flex/cis.flex | 328 + datacat/cup_flex/conf.flex | 66 + datacat/cup_flex/dft.flex | 229 + datacat/cup_flex/finalcoord.cup | 129 + datacat/cup_flex/finalcoord.flex | 151 + datacat/cup_flex/g1.cup | 166 + datacat/cup_flex/g1.flex | 218 + datacat/cup_flex/gaussian.cup | 62 + datacat/cup_flex/gaussian.flex | 267 + datacat/cup_flex/gfinalcoord.cup | 113 + datacat/cup_flex/gfinalcoord.flex | 131 + datacat/cup_flex/gmcscfa.cup | 131 + datacat/cup_flex/gmcscfa.flex | 150 + datacat/cup_flex/gmp2opt.flex | 133 + datacat/cup_flex/gnumatom.cup | 62 + datacat/cup_flex/gnumatom.flex | 71 + datacat/cup_flex/gopt.cup | 120 + datacat/cup_flex/gopt.flex | 197 + datacat/cup_flex/gscfa.cup | 132 + datacat/cup_flex/gscfa.flex | 153 + datacat/cup_flex/gvb.cup | 132 + datacat/cup_flex/gvb.flex | 145 + datacat/cup_flex/hf.flex | 104 + datacat/cup_flex/input.cup | 123 + datacat/cup_flex/input.flex | 153 + datacat/cup_flex/inputm.flex | 153 + datacat/cup_flex/method.cup | 62 + datacat/cup_flex/method.flex | 171 + datacat/cup_flex/method1.flex | 171 + datacat/cup_flex/mfinalcoord.cup | 129 + datacat/cup_flex/mfinalcoord.flex | 151 + datacat/cup_flex/mopt.flex | 169 + datacat/cup_flex/mopta.cup | 132 + datacat/cup_flex/mopta.flex | 179 + datacat/cup_flex/mp2to5a.cup | 27 + datacat/cup_flex/mp2to5a.flex | 172 + datacat/cup_flex/mp2to5b.cup | 27 + datacat/cup_flex/mp2to5b.flex | 187 + datacat/cup_flex/mp2to5c.cup | 27 + datacat/cup_flex/mp2to5c.flex | 174 + datacat/cup_flex/mp2to5d.cup | 63 + datacat/cup_flex/mp2to5d.flex | 198 + datacat/cup_flex/numatom.cup | 99 + datacat/cup_flex/pound.cup | 60 + datacat/cup_flex/pound.flex | 1054 ++ datacat/cup_flex/prepPtplot.flex | 120 + datacat/cup_flex/scfa.cup | 102 + datacat/cup_flex/scfa.flex | 103 + datacat/cup_flex/scfb.cup | 115 + datacat/cup_flex/scfb.flex | 156 + datacat/cup_flex/transformation.flex | 124 + datacat/cup_flex/wavefunction.cup | 62 + datacat/cup_flex/wavefunction.flex | 217 + datacat/cup_flex/wavefunction1.flex | 187 + datacat/cup_flex/whichProgram.cup | 116 + datacat/cup_flex/whichProgram.flex | 78 + datacat/inchi-1 | Bin 0 -> 1065032 bytes datacat/integration-tests/pom.xml | 33 + .../datacat/integration/IntegrationTest.java | 154 + datacat/laravel-pga/vendor/apache/thrift | 1 + datacat/models/pom.xml | 46 + .../apache/airavata/datacat/models/AclDTO.java | 53 + .../airavata/datacat/models/AclFields.java | 27 + .../Messaging/ExperimentOutputCreatedEvent.java | 981 ++ .../Messaging/ExperimentOutputParsedEvent.java | 573 + .../datacat/models/Messaging/Message.java | 814 ++ .../datacat/models/Messaging/MessageLevel.java | 66 + .../datacat/models/Messaging/MessageType.java | 53 + .../airavata/datacat/models/MetadataFields.java | 33 + .../datacat/models/OutputMetadataDTO.java | 142 + .../datacat/models/OutputMonitorMessage.java | 104 + .../models/OutputMonitorMessageType.java | 29 + .../datacat/models/PrimaryQueryParameter.java | 64 + .../datacat/models/PrimaryQueryType.java | 31 + .../airavata/datacat/models/QueryObject.java | 96 + datacat/parsers/pom.xml | 40 + .../airavata/datacat/parsers/DefaultParser.java | 63 + .../airavata/datacat/parsers/IParser.java | 30 + .../parsers/gridchem/B3PW91/B3PW91Lexer.java | 886 ++ .../parsers/gridchem/B3PW91/B3PW91Parser.java | 544 + .../parsers/gridchem/B3PW91/B3PW91Sym.java | 53 + .../parsers/gridchem/GridChemDemoParser.java | 81 + .../parsers/gridchem/GridChemParser.java | 173 + .../parsers/gridchem/GridChemQueueParser.java | 30 + .../datacat/parsers/gridchem/Settings.java | 26 + .../parsers/gridchem/cbsQ/CBSQLexer.java | 879 ++ .../parsers/gridchem/cbsQ/CbsQParser.java | 543 + .../datacat/parsers/gridchem/cbsQ/CbsQSym.java | 53 + .../gridchem/finalcoord/FinalCoordLexer.java | 799 ++ .../gridchem/finalcoord/FinalCoordParser.java | 466 + .../gridchem/finalcoord/FinalCoordSym.java | 48 + .../datacat/parsers/gridchem/g1/G1Lexer.java | 913 ++ .../datacat/parsers/gridchem/g1/G1Parser.java | 539 + .../datacat/parsers/gridchem/g1/G1Sym.java | 54 + .../gridchem/gaussian/GaussianLexer.java | 864 ++ .../gridchem/gaussian/GaussianParser.java | 187 + .../parsers/gridchem/gaussian/GaussianSym.java | 37 + .../gridchem/gfinalcoord/GFinalCoordLexer.java | 773 ++ .../gridchem/gfinalcoord/GFinalCoordParser.java | 339 + .../gridchem/gfinalcoord/GFinalCoordSym.java | 41 + .../parsers/gridchem/gmcscfa/GMCSCFaLexer.java | 782 ++ .../parsers/gridchem/gmcscfa/GMCSCFaSym.java | 45 + .../parsers/gridchem/gmcscfa/GmcscfaParser.java | 336 + .../gridchem/gnumatom/GNumAtomLexer.java | 672 ++ .../gridchem/gnumatom/GNumAtomParser.java | 190 + .../parsers/gridchem/gnumatom/GNumAtomSym.java | 37 + .../parsers/gridchem/gopt/GOPTLexer.java | 797 ++ .../datacat/parsers/gridchem/gopt/GOPTSym.java | 45 + .../parsers/gridchem/gopt/GoptParser.java | 435 + .../parsers/gridchem/gscfa/GSCFaLexer.java | 790 ++ .../parsers/gridchem/gscfa/GSCFaSym.java | 44 + .../parsers/gridchem/gscfa/GscfaParser.java | 336 + .../datacat/parsers/gridchem/gvb/GVBLexer.java | 777 ++ .../datacat/parsers/gridchem/gvb/GVBParser.java | 335 + .../datacat/parsers/gridchem/gvb/GVBSym.java | 45 + .../datacat/parsers/gridchem/inchi/InChI.java | 47 + .../parsers/gridchem/inchi/InChIGenerator.java | 95 + .../parsers/gridchem/input/InputLexer.java | 799 ++ .../parsers/gridchem/input/InputParser.java | 385 + .../parsers/gridchem/input/InputSym.java | 46 + .../parsers/gridchem/method/MethodLexer.java | 752 ++ .../parsers/gridchem/method/MethodParser.java | 185 + .../parsers/gridchem/method/MethodSym.java | 37 + .../gridchem/mfinalcoord/MFinalCoordLexer.java | 793 ++ .../gridchem/mfinalcoord/MFinalCoordParser.java | 408 + .../gridchem/mfinalcoord/MFinalCoordSym.java | 46 + .../parsers/gridchem/mopta/MOptaLexer.java | 830 ++ .../parsers/gridchem/mopta/MOptaParser.java | 388 + .../parsers/gridchem/mopta/MOptaSym.java | 51 + .../parsers/gridchem/mp2to5a/MP2to5aLexer.java | 681 ++ .../parsers/gridchem/mp2to5a/MP2to5aParser.java | 261 + .../parsers/gridchem/mp2to5a/MP2to5aSym.java | 51 + .../parsers/gridchem/mp2to5b/MP2to5bLexer.java | 687 ++ .../parsers/gridchem/mp2to5b/MP2to5bParser.java | 263 + .../parsers/gridchem/mp2to5b/MP2to5bSym.java | 51 + .../parsers/gridchem/mp2to5c/MP2to5cLexer.java | 682 ++ .../parsers/gridchem/mp2to5c/MP2to5cParser.java | 263 + .../parsers/gridchem/mp2to5c/MP2to5cSym.java | 51 + .../parsers/gridchem/mp2to5d/MP2to5dLexer.java | 737 ++ .../parsers/gridchem/mp2to5d/MP2to5dParser.java | 460 + .../parsers/gridchem/mp2to5d/MP2to5dSym.java | 51 + .../parsers/gridchem/scfa/SCFaLexer.java | 703 ++ .../parsers/gridchem/scfa/SCFaParser.java | 322 + .../datacat/parsers/gridchem/scfa/SCFaSym.java | 43 + .../parsers/gridchem/scfb/SCFbLexer.java | 788 ++ .../parsers/gridchem/scfb/SCFbParser.java | 580 + .../datacat/parsers/gridchem/scfb/SCFbSym.java | 71 + .../parsers/gridchem/util/AccessLevels.java | 28 + .../parsers/gridchem/util/Constants.java | 39 + .../datacat/parsers/gridchem/util/Fields.java | 26 + .../gridchem/util/GridChemProperties.java | 77 + .../whichProgram/WhichProgramLexer.java | 642 + .../whichProgram/WhichProgramParser.java | 306 + .../gridchem/whichProgram/WhichProgramSym.java | 43 + .../src/main/resources/gridchem.properties | 16 + .../src/main/resources/regexParser.properties | 9 + .../gridchem/AbstractGridChemParserTest.java | 39 + .../B3PW91/B3PW91GridChemParserTest.java | 62 + .../gridchem/CbsQ/CbsQGridChemParserTest.java | 59 + .../gridchem/G1/G1GridChemParserTest.java | 58 + .../parsers/gridchem/GridChemParserTest.java | 61 + .../MP2to5a/MP2to5aGridChemParserTest.java | 60 + .../MP2to5b/MP2to5bGridChemParserTest.java | 58 + .../MP2to5c/MP2to5cGridChemParserTest.java | 58 + .../gridchem/ParserParameterizedTest.java | 461 + .../FinalCoordParserGridChemParserTest.java | 60 + .../GFinalCoordGridChemParserTest.java | 60 + .../GaussianParserGridChemParserTest.java | 57 + .../gmcscfa/GMCSCFaGridChemParserTest.java | 60 + .../gridchem/gopt/GoptGridChemParserTest.java | 58 + .../gridchem/gscfa/GscfaGridChemParserTest.java | 59 + .../gridchem/gvb/GVBGridChemParserTest.java | 59 + .../inchi/InChIGeneratorGridChemParserTest.java | 68 + .../gridchem/input/InputGridChemParserTest.java | 60 + .../MFinalCoordParserGridChemParserTest.java | 60 + .../gridchem/mopta/MOptaGridChemParserTest.java | 58 + .../mp2to5d/Mp2to5dGridChemParserTest.java | 60 + .../gridchem/scfa/SCFaGridChemParserTest.java | 60 + .../gridchem/scfb/SCFbGridChemParserTest.java | 60 + datacat/pom.xml | 107 + datacat/query.json | 40 + datacat/regexParser/pom.xml | 64 + .../datacat/regexParser/RegexParser.java | 114 + .../regexParser/RegexParserProperties.java | 86 + .../src/main/resources/regexParser.properties | 9 + .../datacat/regexParser/RegexParserTest.java | 57 + datacat/server/pom.xml | 168 + .../server/src/main/assembly/bin-assembly.xml | 77 + .../airavata/datacat/server/DataCatServer.java | 203 + .../airavata/datacat/server/db/IDataModel.java | 48 + .../server/db/solr/SolrClientFactory.java | 49 + .../datacat/server/db/solr/SolrDataModel.java | 83 + .../datacat/server/db/solr/SolrIndexer.java | 107 + .../datacat/server/db/solr/SolrQuerier.java | 133 + .../server/db/solr/SolrQueryBuilder.java | 101 + .../datacat/server/services/DataCatService.java | 168 + .../server/services/PublisherService.java | 112 + .../server/services/UserStoreService.java | 109 + .../server/userstore/IUserStoreClient.java | 29 + .../server/userstore/WSO2IUserStoreClient.java | 102 + .../airavata/datacat/server/util/Constants.java | 22 + .../datacat/server/util/ServerProperties.java | 55 + .../src/main/resources/bin/datacat-server.sh | 94 + datacat/server/src/main/resources/bin/setenv.sh | 65 + .../src/main/resources/conf/server.properties | 34 + .../server/src/main/resources/log4j.properties | 16 + .../resources/security/client-truststore.jks | Bin 0 -> 35240 bytes .../src/main/resources/security/keystore.jks | Bin 0 -> 33260 bytes .../datacat/server/db/SolrIndexerTest.java | 62 + .../datacat/server/db/SolrQuerierTest.java | 177 + datacat/web/portal/dataStoreService.php | 181 + datacat/web/portal/favicon.ico | Bin 0 -> 318 bytes .../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20335 bytes .../fonts/glyphicons-halflings-regular.svg | 229 + .../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 41280 bytes .../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23320 bytes datacat/web/portal/index.php | 95 + .../portal/scripts/bootstrap-datetimepicker.js | 1384 +++ datacat/web/portal/scripts/bootstrap.js | 2114 ++++ datacat/web/portal/scripts/jquery-1.11.1.js | 10308 +++++++++++++++++ datacat/web/portal/scripts/main.js | 610 + datacat/web/portal/scripts/moment.js | 2856 +++++ .../styles/bootstrap-datetimepicker.min.css | 5 + datacat/web/portal/styles/bootstrap-theme.css | 442 + datacat/web/portal/styles/bootstrap.css | 6203 ++++++++++ datacat/web/portal/styles/main.css | 144 + datacat/wso2_is.cer | 14 + 266 files changed, 71201 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4a9b5e3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear + + +target/ +.idea/ +tmp/ +logs/ +gaussian_sample/ +*.ser +*.iml +*.gen + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +solr-server/logs/solr.log + http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/.gitignore ---------------------------------------------------------------------- diff --git a/datacat/.gitignore b/datacat/.gitignore new file mode 100644 index 0000000..7481235 --- /dev/null +++ b/datacat/.gitignore @@ -0,0 +1,22 @@ +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear + +target/ +.idea/ +tmp/ +logs/ +gaussian_sample/ +*.ser +*.iml +*.gen + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +solr-server/logs/solr.log http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/.gitignore.swp ---------------------------------------------------------------------- diff --git a/datacat/.gitignore.swp b/datacat/.gitignore.swp new file mode 100644 index 0000000..05c65af Binary files /dev/null and b/datacat/.gitignore.swp differ http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/LICENSE ---------------------------------------------------------------------- diff --git a/datacat/LICENSE b/datacat/LICENSE new file mode 100644 index 0000000..ad410e1 --- /dev/null +++ b/datacat/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/README.md ---------------------------------------------------------------------- diff --git a/datacat/README.md b/datacat/README.md new file mode 100644 index 0000000..23176b8 --- /dev/null +++ b/datacat/README.md @@ -0,0 +1,23 @@ +DataCat +========= + +DataCat is a scientific data cataloging system designed for efficient and effective management of scientific data. + +This is a project done by students of University of Moratuwa, Sri Lanka as a part of their final year project. The goal is to contribute this code to Apache Airavata project and integrate it with Airavata workflow software and continuing to develop it further. + +The project development team: + * Supun Nakandala â [email protected] + * Sachith Withana - [email protected] + * Dinu Kumarasiri - [email protected] + * Hirantha Sankalpa - [email protected] + +Before Running the Program +========================== +* Add WSO2 server certificate found in the top level directory to the java key store using the below command + $JAVA_HOME/jre/lib/security$ keytool -import -file wso2_is.cer -keystore cacerts + +* Start Solr server using the following command + $java -jar SOLR_HOME/start.jar + +* Start WSO2 IS using the following command + $sh WSO2_IS_HOME/bin/wso2server.sh http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/pom.xml ---------------------------------------------------------------------- diff --git a/datacat/agent/pom.xml b/datacat/agent/pom.xml new file mode 100644 index 0000000..f458d0d --- /dev/null +++ b/datacat/agent/pom.xml @@ -0,0 +1,132 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>datacat</artifactId> + <groupId>org.apache.airavata.datacat</groupId> + <version>1.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <modelVersion>4.0.0</modelVersion> + + <artifactId>agent</artifactId> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>2.3.2</version> + <configuration> + <source>${jdk.version}</source> + <target>${jdk.version}</target> + <compilerArgument>-Xlint:unchecked</compilerArgument> + <optimize>true</optimize> + <encoding>UTF-8</encoding> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.3.2</version> + <configuration> + <archive> + <manifest> + <mainClass>org.apache.airavata.datacat.agent.DataCatAgent</mainClass> + <addClasspath>true</addClasspath> + </manifest> + </archive> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.3</version> + <executions> + <execution> + <id>distribution-package</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <descriptors> + <descriptor>src/main/assembly/bin-assembly.xml</descriptor> + </descriptors> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <dependencies> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-io</artifactId> + <version>${apache-commons.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata.datacat</groupId> + <artifactId>models</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata.datacat</groupId> + <artifactId>parsers</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata.datacat</groupId> + <artifactId>RegexParser</artifactId> + <version>1.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>${httpclient.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <version>${httpcore.version}</version> + </dependency> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>${gson.version}</version> + </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + + <!-- RabbitMQ dependencies - For the message-Broker --> + <dependency> + <groupId>com.rabbitmq</groupId> + <artifactId>amqp-client</artifactId> + <version>${rabbitmq.version}</version> + </dependency> + + <!-- Thrift dependency --> + <dependency> + <groupId>org.apache.thrift</groupId> + <artifactId>libthrift</artifactId> + <version>${thrift.version}</version> + </dependency> + + </dependencies> + + +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/assembly/bin-assembly.xml ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/assembly/bin-assembly.xml b/datacat/agent/src/main/assembly/bin-assembly.xml new file mode 100644 index 0000000..3ca8124 --- /dev/null +++ b/datacat/agent/src/main/assembly/bin-assembly.xml @@ -0,0 +1,77 @@ +<!--Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>bin</id> + <includeBaseDirectory>true</includeBaseDirectory> + <baseDirectory>datacat-agent</baseDirectory> + <formats> + <format>tar.gz</format> + <format>zip</format> + </formats> + + <fileSets> + <fileSet> + <directory>src/main/resources/bin</directory> + <outputDirectory>bin</outputDirectory> + <includes> + <include>*.sh</include> + </includes> + <fileMode>755</fileMode> + </fileSet> + <fileSet> + <directory>src/main/resources/conf</directory> + <outputDirectory>conf</outputDirectory> + <includes> + <include>*.properties</include> + </includes> + </fileSet> + <fileSet> + <directory>../parsers/src/main/resources</directory> + <outputDirectory>conf</outputDirectory> + <includes> + <include>*.properties</include> + </includes> + </fileSet> + <fileSet> + <directory>src/main/resources/security</directory> + <outputDirectory>security</outputDirectory> + <includes> + <include>*.jks</include> + </includes> + </fileSet> + <fileSet> + <directory>src/main/resources</directory> + <outputDirectory>conf</outputDirectory> + <includes> + <include>log4j.properties</include> + </includes> + </fileSet> + </fileSets> + + <dependencySets> + <dependencySet> + <outputDirectory>/lib</outputDirectory> + <includes> + <include>*:jar:*</include> + </includes> + </dependencySet> + </dependencySets> +</assembly> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/DataCatAgent.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/DataCatAgent.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/DataCatAgent.java new file mode 100644 index 0000000..9940c49 --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/DataCatAgent.java @@ -0,0 +1,84 @@ +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ +package org.apache.airavata.datacat.agent; + +import org.apache.airavata.datacat.agent.dispatcher.DispatcherService; +import org.apache.airavata.datacat.agent.messageBroker.AiravataUpdateListener; +import org.apache.airavata.datacat.agent.messageBroker.IMessageBroker; +import org.apache.airavata.datacat.agent.monitor.IMonitor; +import org.apache.airavata.datacat.agent.util.AgentProperties; +import org.apache.airavata.datacat.agent.util.Constants; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.nio.file.Paths; + +public class DataCatAgent { + private static final Logger logger = LogManager.getLogger(DataCatAgent.class); + private IMessageBroker messageBroker; + private DispatcherService dispatcherService; + private IMonitor iMonitor; + + private String monitorType = AgentProperties.getInstance().getProperty(Constants.MONITOR_TYPE,""); + + public DataCatAgent() throws Exception { + dispatcherService = DispatcherService.getInstance(); + if(monitorType.equals("FILE_SYSTEM")){ + iMonitor = new org.apache.airavata.datacat.agent.monitor.impl.LocalFileSystemMonitor(); + }else{ + messageBroker = new AiravataUpdateListener(); + } + } + + public static void main(String[] args) throws Exception { + + final DataCatAgent dataCatAgent = new DataCatAgent(); + dataCatAgent.start(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + public void run() { + logger.info("ShutDown called..."); + dataCatAgent.stop(); + } + }); + } + + public void start() throws Exception { + logger.info("\nStarting DataCat Agent...!\n"); + dispatcherService.startDispatcher(); + if(monitorType.equals("FILE_SYSTEM")){ + String path = AgentProperties.getInstance().getProperty(Constants.DATA_ROOT,""); + iMonitor.startMonitor(Paths.get(path)); + }else{ + messageBroker.startBroker(); + } + } + + public void stop() { + dispatcherService.stopService(); + if(monitorType.equals("FILE_SYSTEM")){ + iMonitor.stopMonitor(); + }else{ + messageBroker.stopBroker(); + } + logger.info("\nGood bye from DataCat agent...!\n"); + } +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/DispatcherService.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/DispatcherService.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/DispatcherService.java new file mode 100644 index 0000000..f3b5c4d --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/DispatcherService.java @@ -0,0 +1,66 @@ +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ +package org.apache.airavata.datacat.agent.dispatcher; + + +import org.apache.airavata.datacat.models.OutputMetadataDTO; +import org.apache.airavata.datacat.models.OutputMonitorMessage; + +public class DispatcherService { + private MetadataDispatcher metadataDispatcher; + private MetadataDispatcherQueue metadataDispatcherQueue; + private MonitorDispatcher monitorDispatcher; + private MonitorDispatcherQueue monitorDispatcherQueue; + + private static DispatcherService instance; + + private DispatcherService() throws Exception { + this.metadataDispatcher = MetadataDispatcher.getInstance(); + this.metadataDispatcherQueue = MetadataDispatcherQueue.getInstance(); + this.monitorDispatcher = MonitorDispatcher.getInstance(); + this.monitorDispatcherQueue = MonitorDispatcherQueue.getInstance(); + } + + public static DispatcherService getInstance() throws Exception { + if(DispatcherService.instance==null){ + DispatcherService.instance = new DispatcherService(); + } + + return DispatcherService.instance; + } + + public void startDispatcher() { + metadataDispatcher.startDispatcher(); + monitorDispatcher.startDispatcher(); + } + + public void stopService() { + } + + public void addFileMetadata(OutputMetadataDTO outputMetadataDTO){ + this.metadataDispatcherQueue.addMetadataToQueue(outputMetadataDTO); + } + + public void addFileMonitorMessage(OutputMonitorMessage outputMonitorMessage){ + this.monitorDispatcherQueue.addMsgToQueue(outputMonitorMessage); + } + +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MetadataDispatcher.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MetadataDispatcher.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MetadataDispatcher.java new file mode 100644 index 0000000..66804b6 --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MetadataDispatcher.java @@ -0,0 +1,178 @@ +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ +package org.apache.airavata.datacat.agent.dispatcher; + +import com.google.gson.Gson; +import org.apache.airavata.datacat.agent.messageBroker.RabbitMQPublisher; +import org.apache.airavata.datacat.agent.util.AgentProperties; +import org.apache.airavata.datacat.agent.util.Constants; +import org.apache.airavata.datacat.models.OutputMetadataDTO; +import org.apache.airavata.datacat.models.Messaging.ExperimentOutputParsedEvent; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLContexts; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClients; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import javax.net.ssl.SSLContext; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.security.KeyStore; + +public class MetadataDispatcher { + + private final Logger logger = LogManager.getLogger(MetadataDispatcher.class); + + private MetadataDispatcherQueue metadataDispatcherQueue = MetadataDispatcherQueue.getInstance(); + + private static MetadataDispatcher instance; + + private SSLConnectionSocketFactory sslsf; + + private RabbitMQPublisher rabbitMQPublisher; + + private MetadataDispatcher() throws Exception { + KeyStore keyStore = KeyStore.getInstance("JKS"); + KeyStore trustStore = KeyStore.getInstance("JKS"); + // set up security context + if (new File("../security/" + AgentProperties.getInstance() + .getProperty(Constants.KEYSTORE_FILE, "")).exists()) { + keyStore.load(new FileInputStream(new File("../security/" + AgentProperties.getInstance() + .getProperty(Constants.KEYSTORE_FILE, ""))),AgentProperties.getInstance() + .getProperty(Constants.KEYSTORE_PWD, "").toCharArray()); + } else {; + keyStore.load(ClassLoader.getSystemResourceAsStream("security/" + + AgentProperties.getInstance().getProperty(Constants.KEYSTORE_FILE, "")), + AgentProperties.getInstance() + .getProperty(Constants.KEYSTORE_PWD, "").toCharArray()); + } + if (new File("../security/" + AgentProperties.getInstance() + .getProperty(Constants.TRUSTSTORE_FILE, "")).exists()) { + keyStore.load(new FileInputStream(new File("../security/" + AgentProperties.getInstance() + .getProperty(Constants.TRUSTSTORE_FILE, ""))),AgentProperties.getInstance() + .getProperty(Constants.TRUSTSTORE_PWD, "").toCharArray()); + } else { + keyStore.load(ClassLoader.getSystemResourceAsStream("security/" + + AgentProperties.getInstance().getProperty(Constants.TRUSTSTORE_FILE, "")), + AgentProperties.getInstance() + .getProperty(Constants.TRUSTSTORE_PWD, "").toCharArray()); + } + SSLContext sslContext = SSLContexts.custom() + .loadKeyMaterial(keyStore,AgentProperties.getInstance() + .getProperty(Constants.KEYSTORE_PWD, "").toCharArray()) + .loadTrustMaterial(trustStore, new TrustSelfSignedStrategy()) + .build(); + + sslsf = new SSLConnectionSocketFactory( + sslContext, new String[] { "TLSv1" }, null, + SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); + + rabbitMQPublisher = new RabbitMQPublisher(); + } + + + public static MetadataDispatcher getInstance() throws Exception { + if(MetadataDispatcher.instance==null){ + MetadataDispatcher.instance = new MetadataDispatcher(); + } + return MetadataDispatcher.instance; + } + + /** + * Starts the message dispatcher + */ + public void startDispatcher() { + (new Thread(new Runnable() { + @Override + public void run() { + OutputMetadataDTO outputMetadataDTO = null; + while (true) { + outputMetadataDTO = metadataDispatcherQueue.getMetadataFromQueue(); + if (outputMetadataDTO != null) { + try { + dispatch(outputMetadataDTO); + } catch (Exception e) { + logger.error(e.toString()); + } + } + } + } + })).start(); + } + + /** + * Dispatch single file metadata + * + * @param outputMetadataDTO + * @throws Exception + */ + private void dispatch(OutputMetadataDTO outputMetadataDTO) throws IOException { + if (outputMetadataDTO != null) { + logger.info("Dispatching newly generated file metadata"); + publishDataToServer(outputMetadataDTO); + + logger.info("Publishing the message to Airavata RabbitMQ Message Broker"); + ExperimentOutputParsedEvent experimentOutputParsedEvent = new ExperimentOutputParsedEvent(); + experimentOutputParsedEvent.setExperimentId(outputMetadataDTO.getExperimentId()); + experimentOutputParsedEvent.setDocumentID(outputMetadataDTO.getId()); + experimentOutputParsedEvent.setStatus("success"); + rabbitMQPublisher.publish(experimentOutputParsedEvent); + } else { + logger.info("Dispatching empty file metadata (doing nothing)"); + } + } + + /** + * Method to publish data to the Publisher REST Service + * @param outputMetadataDTO + */ + private void publishDataToServer(OutputMetadataDTO outputMetadataDTO){ + int count = 0; + while(count<10){ + count++; + String stringUri = ""; + try{ + stringUri = AgentProperties.getInstance().getProperty(Constants.PUBLISHER_ADD_ENDPOINT,""); + HttpClient client = HttpClients.custom().setSSLSocketFactory(sslsf).build(); + HttpPost post = new HttpPost(stringUri); + Gson gson = new Gson(); + StringEntity input = new StringEntity(gson.toJson(outputMetadataDTO));; + post.setEntity(input); + HttpResponse response = client.execute(post); + if (response.getStatusLine().getStatusCode() == 200) { + break; + } + throw new RuntimeException("Failed : HTTP error code : " + + response.getStatusLine().getStatusCode()); + }catch (Exception ex){ + logger.error("Trying to publish (URI:"+stringUri+") for the "+count+" time and failed", ex); + ex.printStackTrace(); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MetadataDispatcherQueue.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MetadataDispatcherQueue.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MetadataDispatcherQueue.java new file mode 100644 index 0000000..35a0745 --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MetadataDispatcherQueue.java @@ -0,0 +1,84 @@ +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ +package org.apache.airavata.datacat.agent.dispatcher; + +import org.apache.airavata.datacat.models.OutputMetadataDTO; + +import java.util.LinkedList; + +public class MetadataDispatcherQueue { + public static MetadataDispatcherQueue instance; + private LinkedList<OutputMetadataDTO> queue; + + private MetadataDispatcherQueue() { + queue = new LinkedList<OutputMetadataDTO>(); + } + + /** + * Returns a singleton dispatch queue object + * + * @return Dispatch queue object + */ + public static MetadataDispatcherQueue getInstance() { + if (instance == null) { + instance = new MetadataDispatcherQueue(); + } + return instance; + } + + /** + * Put a message into the dispatch queue + * + * @param outputMetadataDTO + */ + public synchronized void addMetadataToQueue(OutputMetadataDTO outputMetadataDTO) { + queue.add(outputMetadataDTO); + notifyAll(); + } + + /** + * Returns a FileMetadata object from the dispatcher queue + * + * @return FileMetadata object + */ + public synchronized OutputMetadataDTO getMetadataFromQueue() { + OutputMetadataDTO outputMetadataDTO = queue.pollFirst(); + while (outputMetadataDTO == null) { + + try { + wait(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + outputMetadataDTO = queue.pollFirst(); + } + return outputMetadataDTO; + } + + /** + * Returns dispatch queue size + * + * @return queue size + */ + public long getQueueSize() { + return queue.size(); + } +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MonitorDispatcher.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MonitorDispatcher.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MonitorDispatcher.java new file mode 100644 index 0000000..1666cd4 --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MonitorDispatcher.java @@ -0,0 +1,133 @@ +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ +package org.apache.airavata.datacat.agent.dispatcher; + +import org.apache.airavata.datacat.agent.util.AgentProperties; +import org.apache.airavata.datacat.agent.util.Constants; +import org.apache.airavata.datacat.models.OutputMetadataDTO; +import org.apache.airavata.datacat.models.OutputMonitorMessage; +import org.apache.airavata.datacat.parsers.IParser; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class MonitorDispatcher { + + private final Logger logger = LogManager.getLogger(MonitorDispatcher.class); + + + private MonitorDispatcherQueue monitorDispatcherQueue = MonitorDispatcherQueue.getInstance(); + + + private MetadataDispatcherQueue metadataDispatcherQueue = MetadataDispatcherQueue.getInstance(); + + + private ExecutorService exec; + + + private static MonitorDispatcher instance; + + + private MonitorDispatcher() { + init(); + } + + public static MonitorDispatcher getInstance(){ + if(MonitorDispatcher.instance==null){ + MonitorDispatcher.instance = new MonitorDispatcher(); + } + + return MonitorDispatcher.instance; + } + + private void init() { + try { + exec = Executors.newFixedThreadPool(Integer.parseInt(AgentProperties.getInstance().getProperty( + Constants.MAX_PARSER_THREADS, "1000"))); + } catch (Exception e) { + logger.error(e.toString()); + exec = Executors.newFixedThreadPool(100); + } + } + + /** + * Starts the message dispatcher + */ + public void startDispatcher() { + (new Thread(new Runnable() { + @Override + public void run() { + OutputMonitorMessage directoryUpdateMessage = null; + while (true) { + directoryUpdateMessage = monitorDispatcherQueue.getMsgFromQueue(); + if (directoryUpdateMessage != null) { + try { + dispatch(directoryUpdateMessage); + } catch (Exception e) { + logger.error(e.toString()); + } + } + } + } + })).start(); + } + + /** + * Dispatch single message + * + * @param outputMonitorMessage + * @throws Exception + */ + private void dispatch(final OutputMonitorMessage outputMonitorMessage) throws Exception { + logger.info("Dispatching new message for experiment name: " + outputMonitorMessage.getExperimentName()); + exec.execute(new Runnable() { + @Override + public void run() { + IParser parser = null; + try { + Class c = this.getClass().getClassLoader() + .loadClass(AgentProperties.getInstance().getProperty( + Constants.PARSER_CLASS,"" + )); + parser = (IParser)c.newInstance(); + OutputMetadataDTO outputMetadataDTO = parser.parse(outputMonitorMessage); + metadataDispatcherQueue.addMetadataToQueue(outputMetadataDTO); + logger.info("Successfully parsed the metadata for experiment name:"+ outputMetadataDTO.getExperimentName()); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + logger.error("Cannot load Parser Class." + + " Please check PARSER_CLASS property in agent.properties."); + } catch (InstantiationException e) { + e.printStackTrace(); + logger.error("Cannot load Parser Class." + + " Please check PARSER_CLASS property in agent.properties."); + } catch (IllegalAccessException e) { + e.printStackTrace(); + logger.error("Cannot load Parser Class." + + " Please check PARSER_CLASS property in agent.properties."); + } + } + }); + } + +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MonitorDispatcherQueue.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MonitorDispatcherQueue.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MonitorDispatcherQueue.java new file mode 100644 index 0000000..9a3e37b --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/MonitorDispatcherQueue.java @@ -0,0 +1,84 @@ +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ +package org.apache.airavata.datacat.agent.dispatcher; + +import org.apache.airavata.datacat.models.OutputMonitorMessage; + +import java.util.LinkedList; + +public class MonitorDispatcherQueue { + public static MonitorDispatcherQueue instance; + private LinkedList<OutputMonitorMessage> queue; + + private MonitorDispatcherQueue() { + queue = new LinkedList<OutputMonitorMessage>(); + } + + /** + * Returns a singleton dispatch queue object + * + * @return Dispatch queue object + */ + public static MonitorDispatcherQueue getInstance() { + if (instance == null) { + instance = new MonitorDispatcherQueue(); + } + return instance; + } + + /** + * Put a message into the dispatch queue + * + * @param outputMonitorMessage + */ + public synchronized void addMsgToQueue(OutputMonitorMessage outputMonitorMessage) { + queue.add(outputMonitorMessage); + notifyAll(); + } + + /** + * Returns a DirectoryUpdateMessage object from the dispatcher queue + * + * @return DirectoryUpdateMessage object + */ + public synchronized OutputMonitorMessage getMsgFromQueue() { + OutputMonitorMessage directoryUpdateMessage = queue.pollFirst(); + while (directoryUpdateMessage == null) { + try { + wait(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + directoryUpdateMessage = queue.pollFirst(); + } + return directoryUpdateMessage; + } + + /** + * Returns dispatch queue size + * + * @return queue size + */ + public long getQueueSize() { + return queue.size(); + } + +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/validators/IMetadataValidator.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/validators/IMetadataValidator.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/validators/IMetadataValidator.java new file mode 100644 index 0000000..505bbb7 --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/dispatcher/validators/IMetadataValidator.java @@ -0,0 +1,27 @@ +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ +package org.apache.airavata.datacat.agent.dispatcher.validators; + +import org.apache.airavata.datacat.models.OutputMetadataDTO; + +public interface IMetadataValidator { + public boolean validate(OutputMetadataDTO metadataDTO); +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/AiravataUpdateListener.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/AiravataUpdateListener.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/AiravataUpdateListener.java new file mode 100644 index 0000000..52270bc --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/AiravataUpdateListener.java @@ -0,0 +1,148 @@ +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ +package org.apache.airavata.datacat.agent.messageBroker; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; +import com.rabbitmq.client.QueueingConsumer; +import org.apache.airavata.datacat.agent.dispatcher.MonitorDispatcherQueue; +import org.apache.airavata.datacat.agent.util.AgentProperties; +import org.apache.airavata.datacat.agent.util.Constants; +import org.apache.airavata.datacat.agent.util.ThriftUtils; +import org.apache.airavata.datacat.models.OutputMonitorMessage; +import org.apache.airavata.datacat.models.OutputMonitorMessageType; +import org.apache.airavata.datacat.models.Messaging.ExperimentOutputCreatedEvent; +import org.apache.airavata.datacat.models.Messaging.Message; +import org.apache.airavata.datacat.models.Messaging.MessageType; +import org.apache.log4j.LogManager; +import org.apache.thrift.TBase; + +public class AiravataUpdateListener implements IMessageBroker { + private final org.apache.log4j.Logger logger = LogManager.getLogger(AiravataUpdateListener.class); + + private final String BINDING_KEY; + private final String RABBITMQ_HOST; + private final String EXCHANGE_NAME; + + private boolean runFileUpdateListener = false; + private MonitorDispatcherQueue monitorDispatcherQueue; + + /** + * Initializes the attributes + */ + public AiravataUpdateListener() { + RABBITMQ_HOST = AgentProperties.getInstance().getProperty(Constants.RABBITMQ_HOST, ""); + BINDING_KEY = AgentProperties.getInstance().getProperty(Constants.BINDING_KEY, ""); + EXCHANGE_NAME = AgentProperties.getInstance().getProperty(Constants.EXCHANGE_NAME, ""); + monitorDispatcherQueue = MonitorDispatcherQueue.getInstance(); + runFileUpdateListener = true; + } + + public void startBroker() { + (new Thread(new Runnable() { + @Override + public void run() { + try { + ConnectionFactory factory = new ConnectionFactory(); + factory.setHost(RABBITMQ_HOST); + + Connection connection = factory.newConnection(); + Channel channel = connection.createChannel(); + + channel.exchangeDeclare(EXCHANGE_NAME, "topic"); + String queueName = channel.queueDeclare().getQueue(); + + channel.basicQos(1); + channel.queueBind(queueName, EXCHANGE_NAME, BINDING_KEY); + + logger.debug("Waiting for messages. To exit press CTRL+C"); + + QueueingConsumer consumer = new QueueingConsumer(channel); + channel.basicConsume(queueName, true, consumer); + + while (runFileUpdateListener) { + QueueingConsumer.Delivery delivery = consumer.nextDelivery(); + + Message message = new Message(); + ThriftUtils.createThriftFromBytes(delivery.getBody(), message); + TBase event = null; + + if (message.getMessageType().equals(MessageType.EXPERIMENT_OUTPUT)) { + + ExperimentOutputCreatedEvent experimentOutputCreatedEvent = new ExperimentOutputCreatedEvent(); + ThriftUtils.createThriftFromBytes(message.getEvent(), experimentOutputCreatedEvent); + + logger.debug(" Message Received with message id '" + message.getMessageId() + + "' and with message type '" + message.getMessageType() + "' with experiment name " + + experimentOutputCreatedEvent.getExperimentName()); + + event = experimentOutputCreatedEvent; + + logger.debug(" [x] Received FileInfo Message'"); + process(experimentOutputCreatedEvent, message.getUpdatedTime()); + logger.debug(" [x] Done Processing FileInfo Message"); + } else { + logger.debug("Recieved message of type ..." +message.getMessageType()); + } + } + } catch (Exception e) { + logger.error(e); + } + } + + + })).start(); + + + } + + @Override + public void stopBroker() { + runFileUpdateListener = false; + logger.info("Shutting down FileUpdateListener..."); + } + + /** + * This method processe the Experiment Output Created Event that comes from Airavata + * @param experimentOutputCreatedEvent + * @param modifiedTime + * @throws InterruptedException + */ + private void process(ExperimentOutputCreatedEvent experimentOutputCreatedEvent, long modifiedTime) throws InterruptedException { + if (experimentOutputCreatedEvent != null) { + //creating the output monitor message + OutputMonitorMessage outputMonitorMessage = new OutputMonitorMessage(); + outputMonitorMessage.setExperimentID(experimentOutputCreatedEvent.getExperimentId()); + outputMonitorMessage.setExperimentName(experimentOutputCreatedEvent.getExperimentName()); + outputMonitorMessage.setOutputPath(experimentOutputCreatedEvent.getOutputPath()); + outputMonitorMessage.setOwnerId(experimentOutputCreatedEvent.getOwnerId()); + outputMonitorMessage.setGatewayId(experimentOutputCreatedEvent.getGatewayName()); + outputMonitorMessage.setApplicationName(experimentOutputCreatedEvent.getApplicationName()); + outputMonitorMessage.setHost(experimentOutputCreatedEvent.getHost()); + outputMonitorMessage.setFileMonitorMessageType(OutputMonitorMessageType.FILE_CREATED); + + //Dispatching the message to the queue + monitorDispatcherQueue.addMsgToQueue(outputMonitorMessage); + logger.info("FileMonitor Message Added to the queue output path: " + outputMonitorMessage.getOutputPath()); + } + } +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/IMessageBroker.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/IMessageBroker.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/IMessageBroker.java new file mode 100644 index 0000000..8c18ee5 --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/IMessageBroker.java @@ -0,0 +1,28 @@ +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ +package org.apache.airavata.datacat.agent.messageBroker; + +public interface IMessageBroker { + + public void startBroker() throws Exception; + + public void stopBroker(); +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/RabbitMQProducer.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/RabbitMQProducer.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/RabbitMQProducer.java new file mode 100644 index 0000000..4843d1f --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/RabbitMQProducer.java @@ -0,0 +1,196 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.datacat.agent.messageBroker; + +import com.rabbitmq.client.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class RabbitMQProducer { + public static final int DEFAULT_PRE_FETCH = 64; + + private static Logger log = LoggerFactory.getLogger(RabbitMQProducer.class); + + private Connection connection; + + private Channel channel; + + private QueueingConsumer consumer; + + private String consumerTag; + + private String exchangeName; + + private int prefetchCount = DEFAULT_PRE_FETCH; + + private boolean isReQueueOnFail = false; + + private String url; + + public RabbitMQProducer(String url, String exchangeName) { + this.exchangeName = exchangeName; + this.url = url; + } + + public void setPrefetchCount(int prefetchCount) { + this.prefetchCount = prefetchCount; + } + + public void setReQueueOnFail(boolean isReQueueOnFail) { + this.isReQueueOnFail = isReQueueOnFail; + } + + private void reset() { + consumerTag = null; + } + + private void reInitIfNecessary() throws Exception { + if (consumerTag == null || consumer == null) { + close(); + open(); + } + } + + public void close() { + log.info("Closing channel to exchange {}", exchangeName); + try { + if (channel != null && channel.isOpen()) { + if (consumerTag != null) { + channel.basicCancel(consumerTag); + } + channel.close(); + } + } catch (Exception e) { + log.debug("error closing channel and/or cancelling consumer", e); + } + try { + log.info("closing connection to rabbitmq: " + connection); + connection.close(); + } catch (Exception e) { + log.debug("error closing connection", e); + } + consumer = null; + consumerTag = null; + channel = null; + connection = null; + } + + public void open() throws Exception { + try { + connection = createConnection(); + channel = connection.createChannel(); + if (prefetchCount > 0) { + log.info("setting basic.qos / prefetch count to " + prefetchCount + " for " + exchangeName); + channel.basicQos(prefetchCount); + } + channel.exchangeDeclare(exchangeName, "topic", false); + } catch (Exception e) { + reset(); + String msg = "could not open channel for exchange " + exchangeName; + log.error(msg); + throw new Exception(msg, e); + } + } + + public void send(byte []message, String routingKey) throws Exception { + try { + channel.basicPublish(exchangeName, routingKey, null, message); + } catch (IOException e) { + String msg = "Failed to publish message to exchange: " + exchangeName; + log.error(msg, e); + throw new Exception(msg, e); + } + } + + private Connection createConnection() throws IOException { + try { + ConnectionFactory factory = new ConnectionFactory(); + factory.setHost(url); + Connection connection = factory.newConnection(); + + connection.addShutdownListener(new ShutdownListener() { + public void shutdownCompleted(ShutdownSignalException cause) { + } + }); + log.info("connected to rabbitmq: " + connection + " for " + exchangeName); + return connection; + } catch (Exception e) { + log.info("connection failed to rabbitmq: " + connection + " for " + exchangeName); + return null; + } + } + + public void ackMessage(Long msgId) throws Exception { + try { + channel.basicAck(msgId, false); + } catch (ShutdownSignalException sse) { + reset(); + String msg = "shutdown signal received while attempting to ack message"; + log.error(msg, sse); + throw new Exception(msg, sse); + } catch (Exception e) { + String s = "could not ack for msgId: " + msgId; + log.error(s, e); + throw new Exception(s, e); + } + } + + public void failMessage(Long msgId) throws Exception { + if (isReQueueOnFail) { + failWithRedelivery(msgId); + } else { + deadLetter(msgId); + } + } + + public void failWithRedelivery(Long msgId) throws Exception { + try { + channel.basicReject(msgId, true); + } catch (ShutdownSignalException sse) { + reset(); + String msg = "shutdown signal received while attempting to fail with redelivery"; + log.error(msg, sse); + throw new Exception(msg, sse); + } catch (Exception e) { + String msg = "could not fail with redelivery for msgId: " + msgId; + log.error(msg, e); + throw new Exception(msg, e); + } + } + + public void deadLetter(Long msgId) throws Exception { + try { + channel.basicReject(msgId, false); + } catch (ShutdownSignalException sse) { + reset(); + String msg = "shutdown signal received while attempting to fail with no redelivery"; + log.error(msg, sse); + throw new Exception(msg, sse); + } catch (Exception e) { + String msg = "could not fail with dead-lettering (when configured) for msgId: " + msgId; + log.error(msg, e); + throw new Exception(msg, e); + } + } +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/4231ac35/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/RabbitMQPublisher.java ---------------------------------------------------------------------- diff --git a/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/RabbitMQPublisher.java b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/RabbitMQPublisher.java new file mode 100644 index 0000000..2a1355e --- /dev/null +++ b/datacat/agent/src/main/java/org/apache/airavata/datacat/agent/messageBroker/RabbitMQPublisher.java @@ -0,0 +1,81 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.datacat.agent.messageBroker; + +import org.apache.airavata.datacat.agent.util.AgentProperties; +import org.apache.airavata.datacat.agent.util.Constants; +import org.apache.airavata.datacat.agent.util.ThriftUtils; +import org.apache.airavata.datacat.models.Messaging.ExperimentOutputParsedEvent; +import org.apache.airavata.datacat.models.Messaging.Message; +import org.apache.airavata.datacat.models.Messaging.MessageLevel; +import org.apache.airavata.datacat.models.Messaging.MessageType; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RabbitMQPublisher { + + private static Logger log = LoggerFactory.getLogger(RabbitMQPublisher.class); + private final String BINDING_KEY; + private final String RABBITMQ_HOST; + private final String EXCHANGE_NAME; + + private RabbitMQProducer rabbitMQProducer; + + /** + * Initializing the RabbitMQ related attributes + * @throws Exception + */ + public RabbitMQPublisher() throws Exception { + RABBITMQ_HOST = AgentProperties.getInstance().getProperty(Constants.RABBITMQ_HOST, ""); + BINDING_KEY = AgentProperties.getInstance().getProperty(Constants.BINDING_KEY, ""); + EXCHANGE_NAME = AgentProperties.getInstance().getProperty(Constants.EXCHANGE_NAME, ""); + + rabbitMQProducer = new RabbitMQProducer(RABBITMQ_HOST, EXCHANGE_NAME); + rabbitMQProducer.open(); + } + + /** + * publish the data products to Airavata + * @param experimentOutputParsedEvent + */ + public void publish(ExperimentOutputParsedEvent experimentOutputParsedEvent){ + try { + log.info("Publishing status to datacat rabbitmq..."); + byte[] body = ThriftUtils.serializeThriftObject(experimentOutputParsedEvent); + Message message = new Message(); + message.setEvent(body); + message.setMessageId(experimentOutputParsedEvent.getDocumentID()); + message.setMessageType(MessageType.OUTPUT_PARSED); + message.setUpdatedTime(System.currentTimeMillis()); + String routingKey = experimentOutputParsedEvent.getExperimentId(); + byte[] messageBody = ThriftUtils.serializeThriftObject(message); + rabbitMQProducer.send(messageBody, routingKey); + } catch (TException e) { + String msg = "Error while deserializing the object"; + log.error(msg, e); + } catch (Exception e) { + String msg = "Error while sending to rabbitmq host"; + log.error(msg, e); + } + } +}
