This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git.
from 42c9da6 Non blocking task initial framework
new a4252a9 data orchestrator clients, file listener
new 9971574 data orchestrator api seerver
new 6918b88 Data orchestrator events
new 99d1721 data orchestrator
new 4586509 Fix build errors
new c8c7703 workflow engine api server impl
new 1d71482 Merge branch 'master' of
https://github.com/apache/airavata-data-lake into data_orchestrator
new 049cb5c workflow-engine
new cd91d25 Merge pull request #4 from isururanawaka/data_orchestrator
The 42 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
airavata-data-lake.ipr | 2164 +++++++++++++++++++-
airavata-data-lake.iws | 251 ++-
.../datalake/orchestrator/api/Application.java | 33 -
.../api/controller/HookController.java | 23 -
.../api/controller/TransferController.java | 48 -
.../api/controller/remote/SFTPController.java | 79 -
.../api/db/entity/SFTPCredentialEntity.java | 140 --
.../api/db/entity/SFTPRemoteEntity.java | 92 -
.../api/db/repo/SFTPCredentialRepository.java | 26 -
.../api/db/repo/SFTPRemoteRepository.java | 26 -
.../orchestrator/api/model/TransferEntry.java | 75 -
.../api/model/remote/SFTPCredential.java | 110 -
.../orchestrator/api/model/remote/SFTPRemote.java | 73 -
.../data-orchestrator-clients-core}/pom.xml | 8 +-
.../clients/core/AbstractListener.java | 51 +
.../clients/core/EventPublisher.java | 17 +
.../dataorchestrator/clients/core/Utils.java | 4 +
.../logs/airavata.log | 539 +++++
.../data-orchestrator-file-event-listener/pom.xml | 86 +
.../file/client/FileClientInitializer.java | 51 +
.../file/client/Listener/FileListener.java | 34 +
.../file/client/model/Configuration.java | 84 +
.../file/client/model/FileEvent.java | 10 +
.../file/client/publisher/FileEventPublisher.java | 37 +
.../file/client/watcher/FileWatcher.java | 198 ++
.../file/client/watcher/FileWatcherExecutor.java | 37 +
.../src/main/resources/config.yml | 9 +
.../data-orchestrator-clients}/pom.xml | 16 +-
.../data-orchestrator-messaging}/pom.xml | 13 +-
.../dataorchestrator/messaging/Constants.java | 7 +
.../messaging/MessagingEvents.java | 9 +
.../messaging/consumer/ConsumerCallback.java | 10 +
.../messaging/consumer/MessageConsumer.java | 63 +
.../messaging/model/NotificationEvent.java | 119 ++
.../model/NotificationEventDeserializer.java | 40 +
.../model/NotificationEventSerializer.java | 34 +
.../messaging/publisher/MessageProducer.java | 41 +
.../data-orchestrator-registry/pom.xml | 47 +
.../persistance/DataOrchestratorEntity.java | 162 ++
.../DataOrchestratorEventRepository.java | 6 +
.../registry/persistance/EntityStatus.java | 9 +
.../registry/persistance/WorkflowEntity.java | 91 +
.../persistance/WorkflowEntityRepository.java | 6 +
.../registry/persistance/WorkflowTaskEntity.java | 100 +
.../data-orchestrator-api-server}/pom.xml | 65 +-
.../orchestrator/APIServerInitializer.java | 92 +
.../datalake/orchestrator/Configuration.java | 208 ++
.../db/inmemory/DefaultInMemoryStore.java | 76 +
.../handlers/OrchestratorEventHandler.java | 75 +
.../processor/InboundEventProcessor.java | 83 +
.../processor/OutboundEventProcessor.java | 116 ++
.../src/main/resources/application.properties | 12 +
.../src/main/resources/config.yml | 17 +
.../data-orchestrator-service-core}/pom.xml | 2 +-
.../datalake/orchestrator/core}/AbstractTask.java | 2 +-
.../orchestrator/core/adaptors/StorageAdaptor.java | 17 +
.../core/processor/MessageProcessor.java | 8 +
.../data-orchestrator-service}/pom.xml | 18 +-
data-orchestrator/pom.xml | 6 +-
data-orchestrator/workflow-engine/README.md | 6 +-
data-orchestrator/workflow-engine/pom.xml | 7 +-
.../workflow-engine/workflow-engine-api/pom.xml | 107 +
.../workflow/engine/monitor/AsyncEventMonitor.java | 0
.../engine/services/controller/Controller.java | 0
.../engine/services/handler/APIRunner.java | 27 +
.../services/handler/WorkflowEngineAPIHandler.java | 38 +
.../engine/services/participant/Participant.java | 0
.../services/wm/DataSyncWorkflowManager.java | 0
.../engine/services/wm/PreWorkflowManager.java | 47 +
.../engine/services/wm/WorkflowOperator.java | 0
.../workflow/engine/task/AbstractTask.java | 0
.../workflow/engine/task/BlockingTask.java | 0
.../workflow/engine/task/NonBlockingTask.java | 0
.../orchestrator/workflow/engine/task/OutPort.java | 0
.../workflow/engine/task/TaskParamType.java | 0
.../engine/task/annotation/BlockingTaskDef.java | 0
.../engine/task/annotation/NonBlockingSection.java | 0
.../engine/task/annotation/NonBlockingTaskDef.java | 0
.../engine/task/annotation/TaskOutPort.java | 0
.../workflow/engine/task/annotation/TaskParam.java | 0
.../engine/task/impl/ExampleBlockingTask.java | 0
.../engine/task/impl/ExampleNonBlockingTask.java | 0
.../src/main/resources/application.properties | 15 +-
.../src/main/resources/logback.xml | 0
.../src/main/resources/task-list.yaml | 0
.../workflow-engine-client}/pom.xml | 42 +-
.../engine/client/WorkflowEngineClient.java | 20 +
.../workflow-engine/workflow-engine-stubs}/pom.xml | 9 +-
.../src/main/proto/Common.proto | 8 +-
.../src/main/proto/service/WorkflowService.proto | 26 +-
.../drms-rest-proxy/src/main/resources/drms.pb | Bin 102693 -> 104720
bytes
.../drms-stubs/drms.pb | Bin 102693 -> 104720
bytes
metadata-service/data-builders/data-builders.iml | 2 +-
pom.xml | 5 +-
94 files changed, 5347 insertions(+), 887 deletions(-)
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/HookController.java
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/TransferController.java
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/remote/SFTPController.java
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/db/entity/SFTPCredentialEntity.java
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/db/entity/SFTPRemoteEntity.java
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/db/repo/SFTPCredentialRepository.java
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/db/repo/SFTPRemoteRepository.java
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/model/TransferEntry.java
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/model/remote/SFTPCredential.java
delete mode 100644
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/model/remote/SFTPRemote.java
copy {metadata-service/db-service/client =>
data-orchestrator/data-orchestrator-clients/data-orchestrator-clients-core}/pom.xml
(79%)
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-clients-core/src/main/java/org/apache/airavata/dataorchestrator/clients/core/AbstractListener.java
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-clients-core/src/main/java/org/apache/airavata/dataorchestrator/clients/core/EventPublisher.java
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-clients-core/src/main/java/org/apache/airavata/dataorchestrator/clients/core/Utils.java
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-file-event-listener/logs/airavata.log
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-file-event-listener/pom.xml
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-file-event-listener/src/main/java/org/apache/airavata/dataorchestrator/file/client/FileClientInitializer.java
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-file-event-listener/src/main/java/org/apache/airavata/dataorchestrator/file/client/Listener/FileListener.java
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-file-event-listener/src/main/java/org/apache/airavata/dataorchestrator/file/client/model/Configuration.java
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-file-event-listener/src/main/java/org/apache/airavata/dataorchestrator/file/client/model/FileEvent.java
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-file-event-listener/src/main/java/org/apache/airavata/dataorchestrator/file/client/publisher/FileEventPublisher.java
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-file-event-listener/src/main/java/org/apache/airavata/dataorchestrator/file/client/watcher/FileWatcher.java
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-file-event-listener/src/main/java/org/apache/airavata/dataorchestrator/file/client/watcher/FileWatcherExecutor.java
create mode 100644
data-orchestrator/data-orchestrator-clients/data-orchestrator-file-event-listener/src/main/resources/config.yml
copy {metadata-service/db-service =>
data-orchestrator/data-orchestrator-clients}/pom.xml (72%)
copy {metadata-service/db-service/client =>
data-orchestrator/data-orchestrator-messaging}/pom.xml (69%)
create mode 100644
data-orchestrator/data-orchestrator-messaging/src/main/java/org/apache/airavata/dataorchestrator/messaging/Constants.java
create mode 100644
data-orchestrator/data-orchestrator-messaging/src/main/java/org/apache/airavata/dataorchestrator/messaging/MessagingEvents.java
create mode 100644
data-orchestrator/data-orchestrator-messaging/src/main/java/org/apache/airavata/dataorchestrator/messaging/consumer/ConsumerCallback.java
create mode 100644
data-orchestrator/data-orchestrator-messaging/src/main/java/org/apache/airavata/dataorchestrator/messaging/consumer/MessageConsumer.java
create mode 100644
data-orchestrator/data-orchestrator-messaging/src/main/java/org/apache/airavata/dataorchestrator/messaging/model/NotificationEvent.java
create mode 100644
data-orchestrator/data-orchestrator-messaging/src/main/java/org/apache/airavata/dataorchestrator/messaging/model/NotificationEventDeserializer.java
create mode 100644
data-orchestrator/data-orchestrator-messaging/src/main/java/org/apache/airavata/dataorchestrator/messaging/model/NotificationEventSerializer.java
create mode 100644
data-orchestrator/data-orchestrator-messaging/src/main/java/org/apache/airavata/dataorchestrator/messaging/publisher/MessageProducer.java
create mode 100644 data-orchestrator/data-orchestrator-registry/pom.xml
create mode 100644
data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/DataOrchestratorEntity.java
create mode 100644
data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/DataOrchestratorEventRepository.java
create mode 100644
data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/EntityStatus.java
create mode 100644
data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/WorkflowEntity.java
create mode 100644
data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/WorkflowEntityRepository.java
create mode 100644
data-orchestrator/data-orchestrator-registry/src/main/java/org/apache/airavata/datalake/orchestrator/registry/persistance/WorkflowTaskEntity.java
rename data-orchestrator/{data-orchestrator-api =>
data-orchestrator-service/data-orchestrator-api-server}/pom.xml (55%)
create mode 100644
data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/APIServerInitializer.java
create mode 100644
data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/Configuration.java
create mode 100644
data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/db/inmemory/DefaultInMemoryStore.java
create mode 100644
data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/handlers/OrchestratorEventHandler.java
create mode 100644
data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/InboundEventProcessor.java
create mode 100644
data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/processor/OutboundEventProcessor.java
create mode 100644
data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/application.properties
create mode 100644
data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/config.yml
rename data-orchestrator/{data-orchestrator-core =>
data-orchestrator-service/data-orchestrator-service-core}/pom.xml (87%)
rename
data-orchestrator/{data-orchestrator-core/src/main/java/org/apache/airavata/datalake/orchestrator/core/task
=>
data-orchestrator-service/data-orchestrator-service-core/src/main/java/org/apache/airavata/datalake/orchestrator/core}/AbstractTask.java
(93%)
create mode 100644
data-orchestrator/data-orchestrator-service/data-orchestrator-service-core/src/main/java/org/apache/airavata/datalake/orchestrator/core/adaptors/StorageAdaptor.java
create mode 100644
data-orchestrator/data-orchestrator-service/data-orchestrator-service-core/src/main/java/org/apache/airavata/datalake/orchestrator/core/processor/MessageProcessor.java
copy {metadata-service/db-service =>
data-orchestrator/data-orchestrator-service}/pom.xml (72%)
create mode 100644
data-orchestrator/workflow-engine/workflow-engine-api/pom.xml
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/monitor/AsyncEventMonitor.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/services/controller/Controller.java
(100%)
create mode 100644
data-orchestrator/workflow-engine/workflow-engine-api/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/services/handler/APIRunner.java
create mode 100644
data-orchestrator/workflow-engine/workflow-engine-api/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/services/handler/WorkflowEngineAPIHandler.java
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/services/participant/Participant.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/services/wm/DataSyncWorkflowManager.java
(100%)
create mode 100644
data-orchestrator/workflow-engine/workflow-engine-api/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/services/wm/PreWorkflowManager.java
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/services/wm/WorkflowOperator.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/AbstractTask.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/BlockingTask.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/NonBlockingTask.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/OutPort.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/TaskParamType.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/annotation/BlockingTaskDef.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/annotation/NonBlockingSection.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/annotation/NonBlockingTaskDef.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/annotation/TaskOutPort.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/annotation/TaskParam.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/impl/ExampleBlockingTask.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/task/impl/ExampleNonBlockingTask.java
(100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/resources/application.properties (54%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/resources/logback.xml (100%)
rename data-orchestrator/workflow-engine/{ =>
workflow-engine-api}/src/main/resources/task-list.yaml (100%)
copy {metadata-service/db-service/server =>
data-orchestrator/workflow-engine/workflow-engine-client}/pom.xml (64%)
create mode 100644
data-orchestrator/workflow-engine/workflow-engine-client/src/main/java/org/apache/airavata/datalake/orchestrator/workflow/engine/client/WorkflowEngineClient.java
copy {metadata-service/db-service/stub =>
data-orchestrator/workflow-engine/workflow-engine-stubs}/pom.xml (95%)
copy
data-resource-management-service/drms-stubs/src/main/proto/sharing/Sharing.proto
=>
data-orchestrator/workflow-engine/workflow-engine-stubs/src/main/proto/Common.proto
(87%)
copy data-resource-management-service/drms-stubs/src/main/proto/Common.proto
=>
data-orchestrator/workflow-engine/workflow-engine-stubs/src/main/proto/service/WorkflowService.proto
(63%)
