This is an automated email from the ASF dual-hosted git repository. lahirujayathilake pushed a commit to branch airavata-deployment in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 740ada8f3eb2000569b6f95b072cb397fe8fac39 Author: lahiruj <[email protected]> AuthorDate: Tue Oct 28 12:40:10 2025 -0400 Initial implementation of ansible scripts to deploy and update airavata services --- dev-tools/ansible/airavata_setup.yml | 99 ++++++++ dev-tools/ansible/airavata_update.yml | 68 +++++ dev-tools/ansible/deploy_setup.sh | 72 ++++++ dev-tools/ansible/deploy_update.sh | 75 ++++++ .../inventories/dev/group_vars/all/vars.yml | 76 ++++++ dev-tools/ansible/inventories/dev/hosts | 12 + dev-tools/ansible/inventories/template/README.md | 148 +++++++++++ .../template/group_vars/all/vars.yml.example | 25 ++ .../template/group_vars/all/vault.yml.example | 85 +++++++ .../host_vars/airavata-server/vault.yml.example | 19 ++ .../ansible/inventories/template/hosts.example | 12 + .../roles/airavata_services/defaults/main.yml | 214 ++++++++++++++++ .../roles/airavata_services/handlers/main.yml | 23 ++ .../roles/airavata_services/tasks/build.yml | 66 +++++ .../tasks/deploy_agent_service.yml | 78 ++++++ .../airavata_services/tasks/deploy_api_server.yml | 120 +++++++++ .../airavata_services/tasks/deploy_file_server.yml | 78 ++++++ .../tasks/deploy_research_service.yml | 78 ++++++ .../airavata_services/tasks/deploy_restproxy.yml | 78 ++++++ .../ansible/roles/airavata_services/tasks/main.yml | 36 +++ .../airavata_services/tasks/start_services.yml | 143 +++++++++++ .../airavata_services/tasks/stop_services.yml | 138 +++++++++++ .../templates/airavata-server.properties.j2 | 276 +++++++++++++++++++++ .../templates/application-agent-service.yml.j2 | 63 +++++ .../application-file-server.properties.j2 | 34 +++ .../templates/application-research-service.yml.j2 | 76 ++++++ .../templates/application-restproxy.properties.j2 | 23 ++ .../templates/email-config.yml.j2 | 114 +++++++++ .../airavata_services/templates/log4j2.xml.j2 | 55 ++++ 29 files changed, 2384 insertions(+) diff --git a/dev-tools/ansible/airavata_setup.yml b/dev-tools/ansible/airavata_setup.yml new file mode 100644 index 0000000000..860038a110 --- /dev/null +++ b/dev-tools/ansible/airavata_setup.yml @@ -0,0 +1,99 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Airavata Full Environment Setup Playbook +# +# This playbook sets up a complete Airavata environment from scratch including: +# - Environment setup (users, groups, firewall) +# - Java and Maven installation +# - Zookeeper installation and configuration +# - Kafka installation and configuration +# - RabbitMQ installation and configuration +# - MariaDB installation and configuration +# - SSL certificate setup (Let's Encrypt) +# - Airavata services build and deployment +# +# Usage: +# ansible-playbook -i inventories/<env> airavata_setup.yml --ask-vault-pass +# +# Prerequisites: +# - Clean server with root/sudo access +# - DNS configured (for Let's Encrypt) + +- name: Full Airavata Environment Setup + hosts: airavata_servers + become: yes + + roles: + # Environment and prerequisites + - role: env_setup + tags: + - env_setup + + # Java installation + - role: java + tags: + - java + + # Maven, Git, and source checkout + - role: common + become: yes + become_user: "{{ user | default('airavata') }}" + tags: + - common + + # Zookeeper installation and configuration + - role: zookeeper + tags: + - zookeeper + - airavata + + # Kafka installation and configuration + - role: kafka + tags: + - kafka + - airavata + + # RabbitMQ installation and configuration + - role: rabbitmq + tags: + - rabbitmq + - airavata + + # MariaDB installation and configuration + - role: database + tags: + - database + + # SSL certificates (Let's Encrypt) + - role: letsencrypt + tags: + - ssl + - letsencrypt + + # Build and deploy Airavata services + - role: airavata_services + become: yes + become_user: "{{ user | default('airavata') }}" + tags: + - airavata_services + - deploy + diff --git a/dev-tools/ansible/airavata_update.yml b/dev-tools/ansible/airavata_update.yml new file mode 100644 index 0000000000..1150e4d55d --- /dev/null +++ b/dev-tools/ansible/airavata_update.yml @@ -0,0 +1,68 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Airavata Services Update Playbook +# +# This playbook updates existing Airavata services in environments where +# infrastructure (Zookeeper, RabbitMQ, databases) already exists. +# +# Usage: +# ansible-playbook -i inventories/<env> airavata_update.yml --ask-vault-pass +# +# Prerequisites: +# - Infrastructure components already running (Zookeeper, RabbitMQ, databases) +# - Previous deployment exists +# - SSH access to target servers + +- name: Update Airavata Services + hosts: airavata_servers + become: yes + become_user: "{{ deploy_user | default(ansible_user) }}" + + tasks: + - name: Display deployment information + debug: + msg: "Updating Airavata services on {{ inventory_hostname }}" + + - name: Pull and build Airavata + include_role: + name: airavata_services + tasks_from: build + + - name: Stop all services + include_role: + name: airavata_services + tasks_from: stop_services + + - name: Deploy all services + include_role: + name: airavata_services + tasks_from: main + + - name: Start all services + include_role: + name: airavata_services + tasks_from: start_services + + - name: Display completion message + debug: + msg: "Airavata services update completed on {{ inventory_hostname }}" + diff --git a/dev-tools/ansible/deploy_setup.sh b/dev-tools/ansible/deploy_setup.sh new file mode 100755 index 0000000000..c8c7a6ee63 --- /dev/null +++ b/dev-tools/ansible/deploy_setup.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Full environment setup wrapper script for Airavata deployment + +set -e + +if [ -z "$1" ]; then + echo "Usage: $0 <environment>" + echo "" + echo "Deploys complete Airavata environment from scratch including all infrastructure components." + echo "" + echo "Examples:" + echo " $0 prod" + echo " $0 dev" + echo " $0 staging" + exit 1 +fi + +ENV=$1 +INVENTORY="inventories/$ENV" + +if [ ! -d "$INVENTORY" ]; then + echo "Error: Inventory directory '$INVENTORY' not found" + echo "" + echo "Available inventories:" + ls -1 inventories/ 2>/dev/null || echo " (none found)" + exit 1 +fi + +echo "==========================================" +echo "Full Airavata Setup Deployment" +echo "==========================================" +echo "Environment: $ENV" +echo "Inventory: $INVENTORY" +echo "" +echo "This will set up a complete Airavata environment from scratch," +echo "including Java, Maven, Zookeeper, Kafka, RabbitMQ, MariaDB," +echo "SSL certificates, and all Airavata services." +echo "" +read -p "Continue? (y/N): " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Deployment cancelled." + exit 1 +fi + +echo "Starting deployment..." +ansible-playbook -i "$INVENTORY" airavata_setup.yml --ask-vault-pass + +echo "" +echo "==========================================" +echo "Deployment completed!" +echo "==========================================" + diff --git a/dev-tools/ansible/deploy_update.sh b/dev-tools/ansible/deploy_update.sh new file mode 100755 index 0000000000..7a4bf13de8 --- /dev/null +++ b/dev-tools/ansible/deploy_update.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Service update wrapper script for Airavata deployment + +set -e + +if [ -z "$1" ]; then + echo "Usage: $0 <environment>" + echo "" + echo "Updates existing Airavata services in the specified environment." + echo "" + echo "Examples:" + echo " $0 prod" + echo " $0 dev" + echo "" + exit 1 +fi + +ENV=$1 +INVENTORY="inventories/$ENV" + +if [ ! -d "$INVENTORY" ]; then + echo "Error: Inventory directory '$INVENTORY' not found" + echo "" + echo "Available inventories:" + ls -1 inventories/ 2>/dev/null || echo " (none found)" + exit 1 +fi + +echo "==========================================" +echo "Airavata Service Update" +echo "==========================================" +echo "Environment: $ENV" +echo "Inventory: $INVENTORY" +echo "" +echo "This will:" +echo " 1. Pull latest code from git" +echo " 2. Build Airavata with Maven" +echo " 3. Stop all services" +echo " 4. Deploy new artifacts" +echo " 5. Start all services" +echo "" +read -p "Continue? (y/N): " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Update cancelled." + exit 1 +fi + +echo "Starting update..." +ansible-playbook -i "$INVENTORY" airavata_update.yml --ask-vault-pass + +echo "" +echo "==========================================" +echo "Update completed!" +echo "==========================================" + diff --git a/dev-tools/ansible/inventories/dev/group_vars/all/vars.yml b/dev-tools/ansible/inventories/dev/group_vars/all/vars.yml new file mode 100644 index 0000000000..b964069549 --- /dev/null +++ b/dev-tools/ansible/inventories/dev/group_vars/all/vars.yml @@ -0,0 +1,76 @@ +--- +# Non-sensitive configuration variables for dev environment + +# Airavata version and build settings +airavata_version: "0.21-SNAPSHOT" +git_branch: "master" +airavata_git_repo: "https://github.com/apache/airavata.git"; +airavata_source_dir: "/home/{{ deploy_user }}/airavata-src" +deployment_dir: "/home/{{ deploy_user }}/airavata" + +# Deployment user +deploy_user: "exouser" + +# Service ports +api_server_port: 8930 +profile_service_port: 8962 +registry_port: 8970 +registry_server_port: 8970 +sharing_registry_port: 7878 +cred_store_port: 8960 +agent_service_port: 18880 +research_service_port: 18899 +file_server_port: 8050 +restproxy_port: 8082 + +# Database drivers +registry_jdbc_driver: "org.mariadb.jdbc.Driver" +appcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +replicacatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +workflowcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +sharingcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +profile_service_jdbc_driver: "org.mariadb.jdbc.Driver" +credential_store_jdbc_driver: "org.mariadb.jdbc.Driver" + +# Database users +registry_jdbc_user: "root" +appcatalog_jdbc_user: "root" +replicacatalog_jdbc_user: "root" +workflowcatalog_jdbc_user: "root" +sharingcatalog_jdbc_user: "root" +profile_service_jdbc_user: "root" +credential_store_jdbc_user: "root" + +# Paths +local_data_location: "/home/{{ deploy_user }}/temp-storage" +file_server_storage_location: "/home/{{ deploy_user }}/temp-storage" +agent_service_storage_path: "/var/www/portals/gateway-user-data" + +# Keystore file name +vault_keystore_file: "airavata.sym.p12" + +# Email monitor settings +email_based_monitor_host: "imap.gmail.com" +email_based_monitor_store_protocol: "imaps" +email_based_monitor_folder_name: "INBOX" +email_expiration_minutes: 60 +email_based_monitoring_period: 10000 + +# Kafka and RabbitMQ +kafka_broker_url: "localhost:9092" +restproxy_broker_url: "localhost:9092" +rabbitmq_status_exchange_name: "status_exchange" +rabbitmq_process_exchange_name: "process_exchange" +rabbitmq_experiment_exchange_name: "experiment_exchange" +experiment_launch_queue: "experiment_launch" + +# Zookeeper connection +zookeeper_connection: "localhost:2181" +embedded_zk: false + +# Other non-sensitive configuration +enable_sharing: true +enable_validation: true +enable_realtime_monitor: true +job_notification_enable: true + diff --git a/dev-tools/ansible/inventories/dev/hosts b/dev-tools/ansible/inventories/dev/hosts new file mode 100644 index 0000000000..d0ac8dc5f6 --- /dev/null +++ b/dev-tools/ansible/inventories/dev/hosts @@ -0,0 +1,12 @@ +[airavata_servers] +dev-server + +[zookeeper] +dev-server + +[rabbitmq] +dev-server + +[database] +dev-server + diff --git a/dev-tools/ansible/inventories/template/README.md b/dev-tools/ansible/inventories/template/README.md new file mode 100644 index 0000000000..e82de5cdcd --- /dev/null +++ b/dev-tools/ansible/inventories/template/README.md @@ -0,0 +1,148 @@ +# Airavata Deployment Inventory Template + +This directory contains a template for creating new Airavata deployment inventories. + +## Quick Start for New Environment + +### 1. Copy this template + +```bash +cp -r inventories/template inventories/my-env +cd inventories/my-env +``` + +### 2. Rename example files + +```bash +mv hosts.example hosts +mv group_vars/all/vars.yml.example group_vars/all/vars.yml +mv group_vars/all/vault.yml.example group_vars/all/vault.yml +mv host_vars/airavata-server/vault.yml.example host_vars/airavata-server/vault.yml +``` + +### 3. Edit configuration files + +Edit all files and replace `CHANGEME` values with your actual values: + +**hosts** - Replace `airavata-server` with your host alias if needed +**group_vars/all/vars.yml** - Set non-sensitive configuration values +**group_vars/all/vault.yml** - Set sensitive values (passwords, URLs, etc.) +**host_vars/airavata-server/vault.yml** - Set server IP addresses and SSH credentials + +### 4. Encrypt sensitive files + +Encrypt the vault files to protect sensitive information: + +```bash +cd ../my-env + +# Encrypt group variables (database passwords, API keys, etc.) +ansible-vault encrypt group_vars/all/vault.yml + +# Encrypt host variables (server IPs, SSH keys) +ansible-vault encrypt host_vars/airavata-server/vault.yml +``` + +### 5. Test connection + +Verify you can connect to your server: + +```bash +ansible-playbook -i inventories/my-env --list-hosts -m ping --ask-vault-pass +``` + +### 6. Deploy + +**For initial setup (full environment from scratch):** +```bash +./deploy_setup.sh my-env +``` + +or + +```bash +ansible-playbook -i inventories/my-env airavata_setup.yml --ask-vault-pass +``` + +**For service updates (infrastructure already exists):** +```bash +./deploy_update.sh my-env +``` + +or + +```bash +ansible-playbook -i inventories/my-env airavata_update.yml --ask-vault-pass +``` + +## File Structure + +``` +my-env/ +├── hosts # Host definitions +├── group_vars/ +│ └── all/ +│ ├── vars.yml # Non-sensitive variables +│ └── vault.yml # Encrypted sensitive variables +└── host_vars/ + └── airavata-server/ + └── vault.yml # Encrypted server-specific variables +``` + +## Key Configuration Points + +### Server Access (host_vars/airavata-server/vault.yml) +- `ansible_host` - Server IP address or hostname +- `ansible_user` - SSH user for deployment +- `ansible_ssh_private_key_file` - Path to SSH private key + +### Database Configuration (group_vars/all/vault.yml) +- All database passwords +- Database URLs and connection strings +- Server IP addresses embedded in URLs + +### Service Configuration (group_vars/all/vault.yml) +- IAM/Keycloak credentials +- OAuth client secrets +- RabbitMQ connection strings +- Email monitoring credentials +- Tunnel server tokens +- Keystore passwords + +### Non-Sensitive Configuration (group_vars/all/vars.yml) +- Service ports +- Build settings (git repository, branch, version) +- Paths and directories + +## Managing Vault Files + +**View an encrypted file:** +```bash +ansible-vault view group_vars/all/vault.yml +``` + +**Edit an encrypted file:** +```bash +ansible-vault edit group_vars/all/vault.yml +``` + +**Change vault password:** +```bash +ansible-vault rekey group_vars/all/vault.yml +``` + +## Troubleshooting + +**Issue: Playbook asks for vault password repeatedly** +- Check that all vault files are encrypted +- Verify the inventory directory path is correct + +**Issue: Connection refused** +- Verify `ansible_host` in host_vars is correct +- Check SSH key file path and permissions +- Ensure target server is accessible from your machine + +**Issue: Services don't start** +- Check logs in `deployment_dir/logs/` +- Verify all required ports are open +- Ensure database connectivity \ No newline at end of file diff --git a/dev-tools/ansible/inventories/template/group_vars/all/vars.yml.example b/dev-tools/ansible/inventories/template/group_vars/all/vars.yml.example new file mode 100644 index 0000000000..003ad940b9 --- /dev/null +++ b/dev-tools/ansible/inventories/template/group_vars/all/vars.yml.example @@ -0,0 +1,25 @@ +--- +# Non-sensitive configuration variables +# Copy this file to vars.yml and update values as needed + +# Airavata version and build settings +airavata_version: "0.21-SNAPSHOT" +git_branch: "master" +airavata_git_repo: "https://github.com/apache/airavata.git"; +airavata_source_dir: "/home/{{ deploy_user }}/airavata-src" +deployment_dir: "/home/{{ deploy_user }}/airavata" + +# Non-sensitive service settings +deploy_user: "CHANGEME_DEPLOY_USER" +api_server_port: 8930 +profile_service_port: 8962 +registry_port: 8970 + +# Database info +registry_jdbc_driver: "org.mariadb.jdbc.Driver" +registry_jdbc_user: "root" +appcatalog_jdbc_user: "root" + +# Paths +local_data_location: "/home/{{ deploy_user }}/temp-storage" + diff --git a/dev-tools/ansible/inventories/template/group_vars/all/vault.yml.example b/dev-tools/ansible/inventories/template/group_vars/all/vault.yml.example new file mode 100644 index 0000000000..f738651aeb --- /dev/null +++ b/dev-tools/ansible/inventories/template/group_vars/all/vault.yml.example @@ -0,0 +1,85 @@ +--- +# Sensitive configuration variables (will be encrypted) +# Copy this file to vault.yml, fill in CHANGEME values, then encrypt: +# ansible-vault encrypt vault.yml + +# Database passwords +registry_jdbc_password: "CHANGEME_DB_PASSWORD" +appcatalog_jdbc_password: "CHANGEME_DB_PASSWORD" +replicacatalog_jdbc_password: "CHANGEME_DB_PASSWORD" +workflowcatalog_jdbc_password: "CHANGEME_DB_PASSWORD" +sharingcatalog_jdbc_password: "CHANGEME_DB_PASSWORD" +profile_service_jdbc_password: "CHANGEME_DB_PASSWORD" +credential_store_jdbc_password: "CHANGEME_DB_PASSWORD" + +# Database URLs +registry_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/experiment_catalog" +appcatalog_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/app_catalog" +replicacatalog_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/replica_catalog" +workflowcatalog_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/workflow_catalog" +sharingcatalog_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/sharing_catalog" +profile_service_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/profile_service" +credential_store_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/credential_store" + +# IAM credentials +iam_server_url: "https://CHANGEME_IAM_SERVER"; +iam_admin_username: "admin" +iam_admin_password: "CHANGEME_IAM_PASSWORD" + +# OAuth secrets +default_registry_password: "CHANGEME_REGISTRY_PASSWORD" +default_registry_oauth_client_secret: "CHANGEME_OAUTH_SECRET" + +# RabbitMQ +rabbitmq_broker_url: "amqp://guest:guest@localhost:5672/CHANGEME_VHOST" + +# Zookeeper +zookeeper_connection: "localhost:2181" + +# Email monitoring +email_based_monitor_address: "CHANGEME_EMAIL_ADDRESS" +email_based_monitor_password: "CHANGEME_EMAIL_PASSWORD" + +# Kafka +kafka_broker_url: "localhost:9092" +job_status_publish_endpoint: "http://CHANGEME_API_HOST:8082/topics/helix-airavata-mq"; + +# Tunnel tokens (for agent service) +tunnel_server_host: "CHANGEME_TUNNEL_HOST" +tunnel_server_port: 17000 +tunnel_server_token: "CHANGEME_TUNNEL_TOKEN" +tunnel_server_api_url: "http://CHANGEME_TUNNEL_HOST:8000"; + +# Keystore passwords +keystore_password: "CHANGEME_KEYSTORE_PASSWORD" +credential_store_keystore_password: "CHANGEME_CRED_STORE_PASSWORD" + +# Agent service datasource +agent_service_datasource_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/app_catalog" +agent_service_datasource_username: "root" +agent_service_datasource_password: "CHANGEME_DB_PASSWORD" + +# Research service datasource +research_service_datasource_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/research_catalog" +research_service_datasource_username: "root" +research_service_datasource_password: "CHANGEME_DB_PASSWORD" + +# Agent service Airavata settings +agent_service_airavata_url: "localhost" +agent_service_airavata_port: 8930 +agent_service_airavata_secure: false + +# Research service URLs +research_hub_url: "https://CHANGEME_RESEARCH_HUB"; +research_hub_dev_user: "[email protected]" +research_hub_admin_api_key: "CHANGEME_API_KEY" +research_hub_limit: 10 +research_portal_dev_url: "https://CHANGEME_PORTAL"; +research_portal_url: "https://CHANGEME_PORTAL"; +openid_url: "https://CHANGEME_AUTH_SERVER/realms/default/.well-known/openid-configuration"; +user_profile_server_url: "CHANGEME_PROFILE_SERVER" +user_profile_server_port: 8962 + +# Keystore file +vault_keystore_file: "files/airavata.sym.p12" + diff --git a/dev-tools/ansible/inventories/template/host_vars/airavata-server/vault.yml.example b/dev-tools/ansible/inventories/template/host_vars/airavata-server/vault.yml.example new file mode 100644 index 0000000000..477d50a07e --- /dev/null +++ b/dev-tools/ansible/inventories/template/host_vars/airavata-server/vault.yml.example @@ -0,0 +1,19 @@ +--- +# Server-specific sensitive variables (will be encrypted) +# Copy this file to vault.yml, fill in values, then encrypt: +# ansible-vault encrypt vault.yml + +# SSH connection details +ansible_host: "CHANGEME_SERVER_IP" +ansible_user: "CHANGEME_SSH_USER" +ansible_ssh_private_key_file: "CHANGEME_PATH_TO_SSH_KEY" + +# Optional: If using password-based SSH +# ansible_password: "CHANGEME_SSH_PASSWORD" + +# Optional: If SSH runs on non-standard port +# ansible_port: 22 + +# Optional: SSH connection timeout +# ansible_ssh_timeout: 30 + diff --git a/dev-tools/ansible/inventories/template/hosts.example b/dev-tools/ansible/inventories/template/hosts.example new file mode 100644 index 0000000000..9c1852ea78 --- /dev/null +++ b/dev-tools/ansible/inventories/template/hosts.example @@ -0,0 +1,12 @@ +[airavata_servers] +airavata-server + +[zookeeper] +airavata-server + +[rabbitmq] +airavata-server + +[database] +airavata-server + diff --git a/dev-tools/ansible/roles/airavata_services/defaults/main.yml b/dev-tools/ansible/roles/airavata_services/defaults/main.yml new file mode 100644 index 0000000000..0aacf9f5df --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/defaults/main.yml @@ -0,0 +1,214 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Airavata Services Role Default Variables + +# Service distribution names +airavata_version: "0.21-SNAPSHOT" +api_server_dist_name: "apache-airavata-api-server-{{ airavata_version }}" +agent_service_dist_name: "apache-airavata-agent-service-{{ airavata_version }}" +research_service_dist_name: "apache-airavata-research-service-{{ airavata_version }}" +file_server_dist_name: "apache-airavata-file-server-{{ airavata_version }}" +restproxy_dist_name: "apache-airavata-restproxy-{{ airavata_version }}" + +# Service directories +api_server_dir: "{{ deployment_dir }}/{{ api_server_dist_name }}" +agent_service_dir: "{{ deployment_dir }}/{{ agent_service_dist_name }}" +research_service_dir: "{{ deployment_dir }}/{{ research_service_dist_name }}" +file_server_dir: "{{ deployment_dir }}/{{ file_server_dist_name }}" +restproxy_dir: "{{ deployment_dir }}/{{ restproxy_dist_name }}" + +# Service ports +api_server_port: 8930 +profile_service_port: 8962 +registry_port: 8970 +registry_server_port: 8970 +sharing_registry_port: 7878 +cred_store_port: 8960 +agent_service_port: 18880 +research_service_port: 18899 +file_server_port: 8050 +restproxy_port: 8082 + +# Service hosts +api_server_host: "0.0.0.0" +profile_service_host: "0.0.0.0" +agent_service_server_address: "0.0.0.0" +research_service_server_address: "0.0.0.0" + +# Monitoring ports +api_server_monitoring_port: 9097 +participant_monitoring_port: 9096 +pre_wm_monitoring_port: 9093 +post_wm_monitoring_port: 9094 + +# Monitoring hosts +api_server_monitoring_host: "localhost" +participant_monitoring_host: "localhost" +pre_workflow_manager_monitoring_host: "localhost" +post_workflow_manager_monitoring_host: "localhost" + +# Server hosts +orchestrator_server_host: "localhost" +regserver_server_host: "localhost" +sharing_registry_server_host: "localhost" +cred_store_server_host: "localhost" + +# Orchestrator configuration +orchestrator_class: "org.apache.airavata.orchestrator.server.OrchestratorServer" +orchestrator_server_port: 8940 +orchestrator_server_min_threads: 50 +job_validators: "org.apache.airavata.orchestrator.core.validator.impl.BatchQueueValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator" +enable_validation: true +host_scheduler: "org.apache.airavata.orchestrator.core.schedule.DefaultHostScheduler" + +# Registry server configuration +regserver_class: "org.apache.airavata.registry.api.service.RegistryAPIServer" + +# Sharing registry configuration +sharing_server_class: "org.apache.airavata.sharing.registry.server.SharingRegistryServer" +enable_sharing: true + +# Default registry user +default_registry_user: "default-admin" +default_registry_gateway: "default" +default_registry_oauth_client_id: "pga" +super_tenant_gatewayId: "default" + +# JDBC driver +registry_jdbc_driver: "org.mariadb.jdbc.Driver" +appcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +replicacatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +workflowcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +sharingcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +profile_service_jdbc_driver: "org.mariadb.jdbc.Driver" +credential_store_jdbc_driver: "org.mariadb.jdbc.Driver" + +# Security configuration +security_manager_class: "org.apache.airavata.service.security.KeyCloakSecurityManager" +TLS_enabled: false +TLS_client_timeout: 10000 +keystore_path: "keystores/airavata.p12" +authz_cache_enabled: true +authz_cache_manager_class: "org.apache.airavata.service.security.authzcache.DefaultAuthzCacheManager" +in_memory_cache_size: 1000 + +# Keystore configuration +credential_store_keystore_url: "keystores/airavata.sym.p12" +credential_store_keystore_alias: "airavata" + +# Job notification configuration +job_notification_enable: true +job_notification_emailids: "" +prefetch_count: 200 +durable_queue: false + +# RabbitMQ configuration +rabbitmq_status_exchange_name: "status_exchange" +rabbitmq_process_exchange_name: "process_exchange" +rabbitmq_experiment_exchange_name: "experiment_exchange" +experiment_launch_queue: "experiment_launch" + +# Zookeeper configuration +embedded_zk: false + +# Helix configuration +helix_cluster_name: "AiravataCluster" +helix_controller_name: "AiravataController" +helix_participant_name: "AiravataParticipant" +participant_monitoring_enabled: true + +# Job monitor configuration +enable_realtime_monitor: true +realtime_monitor_broker_consumer_group: "monitor" +realtime_monitor_broker_topic: "helix-airavata-mq" +job_monitor_broker_consumer_group: "MonitoringConsumer" +job_monitor_broker_topic: "monitoring-data" +job_monitor_broker_publisher_id: "AiravataMonitorPublisher" +job_monitor_email_publisher_id: "EmailBasedProducer" +job_monitor_realtime_publisher_id: "RealtimeProducer" + +# Email monitor configuration +email_based_monitor_host: "imap.gmail.com" +email_based_monitor_store_protocol: "imaps" +email_based_monitor_folder_name: "INBOX" +email_expiration_minutes: 60 +email_based_monitoring_period: 10000 + +# Pre-workflow manager configuration +pre_workflow_manager_loadbalance_clusters: false +pre_workflow_manager_monitoring_enabled: true +pre_workflow_manager_name: "AiravataPreWM" + +# Post-workflow manager configuration +post_workflow_manager_loadbalance_clusters: false +post_workflow_manager_monitoring_enabled: true +post_workflow_manager_name: "AiravataPostWM" + +# Parser-workflow configuration +data_parser_delete_container: true +data_parser_broker_consumer_group: "CHANGE_ME" +data_parser_topic: "CHANGE_ME" +data_parser_storage_resource_id: "CHANGE_ME" + +# Monitoring and scanning configuration +cluster_status_monitoring_enable: false +metaschedluer_job_scanning_enable: false +data_analyzer_job_scanning_enable: false + +# Data staging configuration +enable_streaming_transfer: false + +# Thrift client pool configuration +thrift_client_pool_abandoned_removal_enabled: true +thrift_client_pool_abandoned_removal_logged: false + +# DB Event Manager +db_event_manager_class: "org.apache.airavata.db.event.manager.DBEventManagerRunner" + +# Agent service configuration +agent_service_grpc_host: "api.dev.cybershuttle.org" +agent_service_grpc_port: 19900 +agent_service_grpc_max_inbound_message_size: 10485760 +agent_service_max_file_size: "200MB" +agent_service_max_request_size: "200MB" +agent_service_pool_name: "AppCatalogPool" +agent_service_leak_detection_threshold: 20000 +agent_service_ddl_auto: "create" + +# Research service configuration +research_service_grpc_port: 19908 +research_service_max_file_size: "200MB" +research_service_max_request_size: "200MB" +research_service_pool_name: "ResearchCatalogPool" +research_service_leak_detection_threshold: 20000 +research_service_ddl_auto: "validate" + +# File server configuration +file_server_max_file_size: "12GB" +file_server_max_request_size: "12GB" +file_server_file_size_threshold: "2MB" + +# Storage configuration +local_data_location: "/home/{{ deploy_user }}/temp-storage" +agent_service_storage_resource_id: "default_9c15d8af-3d36-4c3c-a07a-0f3b4bb5b903" +agent_service_storage_path: "/var/www/portals/gateway-user-data" +agent_service_application_interface_id: "AiravataAgent_3eeb580b-b0c6-4f7e-8e3d-22c4f84ec3f1" diff --git a/dev-tools/ansible/roles/airavata_services/handlers/main.yml b/dev-tools/ansible/roles/airavata_services/handlers/main.yml new file mode 100644 index 0000000000..087ccde391 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/handlers/main.yml @@ -0,0 +1,23 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Handlers for Airavata Services role + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/build.yml b/dev-tools/ansible/roles/airavata_services/tasks/build.yml new file mode 100644 index 0000000000..310492fcd3 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/build.yml @@ -0,0 +1,66 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Pull latest code from git + git: + repo: "{{ airavata_git_repo }}" + dest: "{{ airavata_source_dir }}" + version: "{{ git_branch }}" + update: yes + register: git_pull_result + tags: + - build + +- name: Display git pull result + debug: + msg: "Git pull completed. Changed: {{ git_pull_result.changed }}" + +- name: Build Airavata with Maven + command: mvn clean install -DskipTests + args: + chdir: "{{ airavata_source_dir }}" + environment: + MAVEN_OPTS: "-Xmx2048m" + register: maven_build_result + tags: + - build + +- name: Display maven build result + debug: + msg: "Maven build completed successfully" + +- name: Check for built distributions + find: + paths: "{{ airavata_source_dir }}/distribution" + patterns: "apache-airavata-*.tar.gz" + register: distribution_files + tags: + - build + +- name: Verify required distribution files exist + assert: + that: + - distribution_files.matched >= 5 + fail_msg: "Expected at least 5 distribution files, found {{ distribution_files.matched }}" + success_msg: "Found {{ distribution_files.matched }} distribution files" + tags: + - build + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/deploy_agent_service.yml b/dev-tools/ansible/roles/airavata_services/tasks/deploy_agent_service.yml new file mode 100644 index 0000000000..6e603411f9 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/deploy_agent_service.yml @@ -0,0 +1,78 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Remove old Agent Service deployment if exists + file: + path: "{{ agent_service_dir }}" + state: absent + tags: + - deploy + - agent-service + +- name: Extract Agent Service distribution + unarchive: + src: "{{ airavata_source_dir }}/distribution/apache-airavata-agent-service-{{ airavata_version }}.tar.gz" + dest: "{{ deployment_dir }}" + remote_src: yes + tags: + - deploy + - agent-service + +- name: Create configuration directory for Agent Service + file: + path: "{{ agent_service_dir }}/conf" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - agent-service + +- name: Deploy application.yml for Agent Service + template: + src: application-agent-service.yml.j2 + dest: "{{ agent_service_dir }}/conf/application.yml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - agent-service + +- name: Deploy log4j2.xml for Agent Service + template: + src: log4j2.xml.j2 + dest: "{{ agent_service_dir }}/conf/log4j2.xml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - agent-service + +- name: Set executable permissions on Agent Service script + file: + path: "{{ agent_service_dir }}/bin/agent-service.sh" + mode: "0755" + tags: + - deploy + - agent-service + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/deploy_api_server.yml b/dev-tools/ansible/roles/airavata_services/tasks/deploy_api_server.yml new file mode 100644 index 0000000000..b93cf52775 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/deploy_api_server.yml @@ -0,0 +1,120 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Remove old API Server deployment if exists + file: + path: "{{ api_server_dir }}" + state: absent + tags: + - deploy + - api-server + +- name: Extract API Server distribution + unarchive: + src: "{{ airavata_source_dir }}/distribution/apache-airavata-api-server-{{ airavata_version }}.tar.gz" + dest: "{{ deployment_dir }}" + remote_src: yes + tags: + - deploy + - api-server + +- name: Create configuration directory for API Server + file: + path: "{{ api_server_dir }}/conf" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - api-server + +- name: Create keystores directory for API Server + file: + path: "{{ api_server_dir }}/conf/keystores" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - api-server + +- name: Deploy airavata-server.properties + template: + src: airavata-server.properties.j2 + dest: "{{ api_server_dir }}/conf/airavata-server.properties" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - api-server + +- name: Deploy email-config.yml + template: + src: email-config.yml.j2 + dest: "{{ api_server_dir }}/conf/email-config.yml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - api-server + +- name: Deploy log4j2.xml + template: + src: log4j2.xml.j2 + dest: "{{ api_server_dir }}/conf/log4j2.xml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - api-server + +- name: Copy keystore file + copy: + src: "{{ inventory_dir }}/files/{{ vault_keystore_file }}" + dest: "{{ api_server_dir }}/conf/keystores/{{ vault_keystore_file | basename }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0600" + decrypt: yes + when: vault_keystore_file is defined + tags: + - deploy + - api-server + +- name: Set executable permissions on API Server scripts + file: + path: "{{ api_server_dir }}/bin/{{ item }}" + mode: "0755" + loop: + - orchestrator.sh + - controller.sh + - participant.sh + - pre-wm.sh + - post-wm.sh + - email-monitor.sh + - realtime-monitor.sh + tags: + - deploy + - api-server + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/deploy_file_server.yml b/dev-tools/ansible/roles/airavata_services/tasks/deploy_file_server.yml new file mode 100644 index 0000000000..99bb583f85 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/deploy_file_server.yml @@ -0,0 +1,78 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Remove old File Server deployment if exists + file: + path: "{{ file_server_dir }}" + state: absent + tags: + - deploy + - file-server + +- name: Extract File Server distribution + unarchive: + src: "{{ airavata_source_dir }}/distribution/apache-airavata-file-server-{{ airavata_version }}.tar.gz" + dest: "{{ deployment_dir }}" + remote_src: yes + tags: + - deploy + - file-server + +- name: Create configuration directory for File Server + file: + path: "{{ file_server_dir }}/conf" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - file-server + +- name: Deploy application.properties for File Server + template: + src: application-file-server.properties.j2 + dest: "{{ file_server_dir }}/conf/application.properties" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - file-server + +- name: Deploy log4j2.xml for File Server + template: + src: log4j2.xml.j2 + dest: "{{ file_server_dir }}/conf/log4j2.xml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - file-server + +- name: Set executable permissions on File Server script + file: + path: "{{ file_server_dir }}/bin/file-service.sh" + mode: "0755" + tags: + - deploy + - file-server + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/deploy_research_service.yml b/dev-tools/ansible/roles/airavata_services/tasks/deploy_research_service.yml new file mode 100644 index 0000000000..132f3e715c --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/deploy_research_service.yml @@ -0,0 +1,78 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Remove old Research Service deployment if exists + file: + path: "{{ research_service_dir }}" + state: absent + tags: + - deploy + - research-service + +- name: Extract Research Service distribution + unarchive: + src: "{{ airavata_source_dir }}/distribution/apache-airavata-research-service-{{ airavata_version }}.tar.gz" + dest: "{{ deployment_dir }}" + remote_src: yes + tags: + - deploy + - research-service + +- name: Create configuration directory for Research Service + file: + path: "{{ research_service_dir }}/conf" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - research-service + +- name: Deploy application.yml for Research Service + template: + src: application-research-service.yml.j2 + dest: "{{ research_service_dir }}/conf/application.yml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - research-service + +- name: Deploy log4j2.xml for Research Service + template: + src: log4j2.xml.j2 + dest: "{{ research_service_dir }}/conf/log4j2.xml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - research-service + +- name: Set executable permissions on Research Service script + file: + path: "{{ research_service_dir }}/bin/research-service.sh" + mode: "0755" + tags: + - deploy + - research-service + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/deploy_restproxy.yml b/dev-tools/ansible/roles/airavata_services/tasks/deploy_restproxy.yml new file mode 100644 index 0000000000..0aeaa94a0e --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/deploy_restproxy.yml @@ -0,0 +1,78 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Remove old REST Proxy deployment if exists + file: + path: "{{ restproxy_dir }}" + state: absent + tags: + - deploy + - restproxy + +- name: Extract REST Proxy distribution + unarchive: + src: "{{ airavata_source_dir }}/distribution/apache-airavata-restproxy-{{ airavata_version }}.tar.gz" + dest: "{{ deployment_dir }}" + remote_src: yes + tags: + - deploy + - restproxy + +- name: Create configuration directory for REST Proxy + file: + path: "{{ restproxy_dir }}/conf" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - restproxy + +- name: Deploy application.properties for REST Proxy + template: + src: application-restproxy.properties.j2 + dest: "{{ restproxy_dir }}/conf/application.properties" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - restproxy + +- name: Deploy log4j2.xml for REST Proxy + template: + src: log4j2.xml.j2 + dest: "{{ restproxy_dir }}/conf/log4j2.xml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - restproxy + +- name: Set executable permissions on REST Proxy script + file: + path: "{{ restproxy_dir }}/bin/restproxy.sh" + mode: "0755" + tags: + - deploy + - restproxy + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/main.yml b/dev-tools/ansible/roles/airavata_services/tasks/main.yml new file mode 100644 index 0000000000..84f10aec2a --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/main.yml @@ -0,0 +1,36 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Include service deployment tasks + include_tasks: deploy_api_server.yml + +- name: Include agent service deployment tasks + include_tasks: deploy_agent_service.yml + +- name: Include research service deployment tasks + include_tasks: deploy_research_service.yml + +- name: Include file server deployment tasks + include_tasks: deploy_file_server.yml + +- name: Include REST proxy deployment tasks + include_tasks: deploy_restproxy.yml + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/start_services.yml b/dev-tools/ansible/roles/airavata_services/tasks/start_services.yml new file mode 100644 index 0000000000..ea74081310 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/start_services.yml @@ -0,0 +1,143 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Start API Server orchestrator + shell: "{{ api_server_dir }}/bin/orchestrator.sh -d start api-orch" + args: + chdir: "{{ api_server_dir }}" + tags: + - start + - api-server + +- name: Start API Server controller + shell: "{{ api_server_dir }}/bin/controller.sh -d start" + args: + chdir: "{{ api_server_dir }}" + tags: + - start + - api-server + +- name: Start API Server participant + shell: "{{ api_server_dir }}/bin/participant.sh -d start" + args: + chdir: "{{ api_server_dir }}" + tags: + - start + - api-server + +- name: Start API Server email-monitor + shell: "{{ api_server_dir }}/bin/email-monitor.sh -d start" + args: + chdir: "{{ api_server_dir }}" + tags: + - start + - api-server + +- name: Start API Server realtime-monitor + shell: "{{ api_server_dir }}/bin/realtime-monitor.sh -d start" + args: + chdir: "{{ api_server_dir }}" + tags: + - start + - api-server + +- name: Start API Server pre-wm + shell: "{{ api_server_dir }}/bin/pre-wm.sh -d start" + args: + chdir: "{{ api_server_dir }}" + tags: + - start + - api-server + +- name: Start API Server post-wm + shell: "{{ api_server_dir }}/bin/post-wm.sh -d start" + args: + chdir: "{{ api_server_dir }}" + tags: + - start + - api-server + +- name: Wait for API Server to start + wait_for: + timeout: 30 + tags: + - start + - api-server + +- name: Start Agent Service + shell: "{{ agent_service_dir }}/bin/agent-service.sh -d start" + args: + chdir: "{{ agent_service_dir }}" + tags: + - start + - agent-service + +- name: Wait for Agent Service to start + wait_for: + timeout: 30 + tags: + - start + - agent-service + +- name: Start Research Service + shell: "{{ research_service_dir }}/bin/research-service.sh -d start" + args: + chdir: "{{ research_service_dir }}" + tags: + - start + - research-service + +- name: Wait for Research Service to start + wait_for: + timeout: 30 + tags: + - start + - research-service + +- name: Start File Server + shell: "{{ file_server_dir }}/bin/file-service.sh -d start" + args: + chdir: "{{ file_server_dir }}" + tags: + - start + - file-server + +- name: Wait for File Server to start + wait_for: + timeout: 30 + tags: + - start + - file-server + +- name: Start REST Proxy + shell: "{{ restproxy_dir }}/bin/restproxy.sh -d start" + args: + chdir: "{{ restproxy_dir }}" + tags: + - start + - restproxy + +- name: Wait for REST Proxy to start + wait_for: + timeout: 30 + tags: + - start + - restproxy diff --git a/dev-tools/ansible/roles/airavata_services/tasks/stop_services.yml b/dev-tools/ansible/roles/airavata_services/tasks/stop_services.yml new file mode 100644 index 0000000000..9bf784df76 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/stop_services.yml @@ -0,0 +1,138 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Check if API Server binary exists + stat: + path: "{{ api_server_dir }}/bin/orchestrator.sh" + register: api_server_exists + tags: + - stop + +- name: Stop API Server orchestrator if exists + shell: "{{ api_server_dir }}/bin/orchestrator.sh -d stop api-orch" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server controller if exists + shell: "{{ api_server_dir }}/bin/controller.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server participant if exists + shell: "{{ api_server_dir }}/bin/participant.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server pre-wm if exists + shell: "{{ api_server_dir }}/bin/pre-wm.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server post-wm if exists + shell: "{{ api_server_dir }}/bin/post-wm.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server email-monitor if exists + shell: "{{ api_server_dir }}/bin/email-monitor.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server realtime-monitor if exists + shell: "{{ api_server_dir }}/bin/realtime-monitor.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Check if Agent Service binary exists + stat: + path: "{{ agent_service_dir }}/bin/agent-service.sh" + register: agent_service_exists + tags: + - stop + +- name: Stop Agent Service if exists + shell: "{{ agent_service_dir }}/bin/agent-service.sh -d stop" + when: agent_service_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Check if Research Service binary exists + stat: + path: "{{ research_service_dir }}/bin/research-service.sh" + register: research_service_exists + tags: + - stop + +- name: Stop Research Service if exists + shell: "{{ research_service_dir }}/bin/research-service.sh -d stop" + when: research_service_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Check if File Server binary exists + stat: + path: "{{ file_server_dir }}/bin/file-service.sh" + register: file_server_exists + tags: + - stop + +- name: Stop File Server if exists + shell: "{{ file_server_dir }}/bin/file-service.sh -d stop" + when: file_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Check if REST Proxy binary exists + stat: + path: "{{ restproxy_dir }}/bin/restproxy.sh" + register: restproxy_exists + tags: + - stop + +- name: Stop REST Proxy if exists + shell: "{{ restproxy_dir }}/bin/restproxy.sh -d stop" + when: restproxy_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Wait for services to fully stop + wait_for: + timeout: 30 + tags: + - stop diff --git a/dev-tools/ansible/roles/airavata_services/templates/airavata-server.properties.j2 b/dev-tools/ansible/roles/airavata_services/templates/airavata-server.properties.j2 new file mode 100644 index 0000000000..ac38f4818a --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/airavata-server.properties.j2 @@ -0,0 +1,276 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +########################################################################### +# +# This properties file provides configuration for all Airavata Services: +# +########################################################################### + +########################################################################### +# API Server Registry DB Configuration +########################################################################### +registry.jdbc.driver={{ registry_jdbc_driver }} +registry.jdbc.url={{ registry_jdbc_url }} +registry.jdbc.user={{ registry_jdbc_user }} +registry.jdbc.password={{ registry_jdbc_password }} +validationQuery=SELECT 1 from CONFIGURATION + +########################################################################### +# Application Catalog DB Configuration +########################################################################### +appcatalog.jdbc.driver={{ appcatalog_jdbc_driver }} +appcatalog.jdbc.url={{ appcatalog_jdbc_url }} +appcatalog.jdbc.user={{ appcatalog_jdbc_user }} +appcatalog.jdbc.password={{ appcatalog_jdbc_password }} +appcatalog.validationQuery=SELECT 1 from CONFIGURATION + +########################################################################## +# Replica Catalog DB Configuration +########################################################################### +replicacatalog.jdbc.driver={{ replicacatalog_jdbc_driver }} +replicacatalog.jdbc.url={{ replicacatalog_jdbc_url }} +replicacatalog.jdbc.user={{ replicacatalog_jdbc_user }} +replicacatalog.jdbc.password={{ replicacatalog_jdbc_password }} +replicacatalog.validationQuery=SELECT 1 from CONFIGURATION + +########################################################################### +# Workflow Catalog DB Configuration +########################################################################### +workflowcatalog.jdbc.driver={{ workflowcatalog_jdbc_driver }} +workflowcatalog.jdbc.url={{ workflowcatalog_jdbc_url }} +workflowcatalog.jdbc.user={{ workflowcatalog_jdbc_user }} +workflowcatalog.jdbc.password={{ workflowcatalog_jdbc_password }} +workflowcatalog.validationQuery=SELECT 1 from CONFIGURATION + +########################################################################### +# Sharing Catalog DB Configuration +########################################################################### +sharingcatalog.jdbc.driver={{ sharingcatalog_jdbc_driver }} +sharingcatalog.jdbc.url={{ sharingcatalog_jdbc_url }} +sharingcatalog.jdbc.user={{ sharingcatalog_jdbc_user }} +sharingcatalog.jdbc.password={{ sharingcatalog_jdbc_password }} +sharingcatalog.validationQuery=SELECT 1 from CONFIGURATION + +########################################################################### +# Generic Server Configurations +########################################################################### +enable.sharing={{ enable_sharing }} + +########################################################################### +# Sharing Registry Server Configuration +########################################################################### +sharing_server={{ sharing_server_class }} +sharing.registry.server.host={{ sharing_registry_server_host }} +sharing.registry.server.port={{ sharing_registry_port }} + +########################################################################### +# Registry Server Configurations +########################################################################### +regserver={{ regserver_class }} +regserver.server.host={{ regserver_server_host }} +regserver.server.port={{ registry_server_port }} +regserver.server.min.threads={{ orchestrator_server_min_threads }} + +default.registry.user={{ default_registry_user }} +default.registry.password={{ default_registry_password }} +default.registry.gateway={{ default_registry_gateway }} +default.registry.oauth.client.id={{ default_registry_oauth_client_id }} +default.registry.oauth.client.secret={{ default_registry_oauth_client_secret }} +super.tenant.gatewayId={{ super_tenant_gatewayId }} + +########################################################################### +# API Server Configurations +########################################################################### +apiserver.class={{ apiserver_class }} +apiserver.host={{ api_server_host }} +apiserver.port={{ api_server_port }} +api.server.monitoring.enabled={{ api_server_monitoring_enabled }} +api.server.monitoring.host={{ api_server_monitoring_host }} +api.server.monitoring.port={{ api_server_monitoring_port }} + +########################################################################### +# Orchestrator Server Configurations +########################################################################### +orchestrator={{ orchestrator_class }} +orchestrator.server.host={{ orchestrator_server_host }} +orchestrator.server.port={{ orchestrator_server_port }} +orchestrator.server.min.threads={{ orchestrator_server_min_threads }} + +job.validators={{ job_validators }} +enable.validation={{ enable_validation }} +host.scheduler={{ host_scheduler }} + +########################################################################### +# Job Monitor Configurations +########################################################################### +job.notification.enable={{ job_notification_enable }} +#Provide comma separated email ids as a string if more than one +job.notification.emailids={{ job_notification_emailids }} +job.status.publish.endpoint={{ job_status_publish_endpoint }} + +########################################################################### +# Credential Store module Configuration +########################################################################### +credential.store.keystore.url={{ credential_store_keystore_url }} +credential.store.keystore.alias={{ credential_store_keystore_alias }} +credential.store.keystore.password={{ credential_store_keystore_password }} +credential.store.jdbc.url={{ credential_store_jdbc_url }} +credential.store.jdbc.user={{ credential_store_jdbc_user }} +credential.store.jdbc.password={{ credential_store_jdbc_password }} +credential.store.jdbc.driver={{ credential_store_jdbc_driver }} +credential.store.server.host={{ cred_store_server_host }} +credential.store.server.port={{ cred_store_port }} +credential.store.class={{ credential_store_class }} +credential.store.jdbc.validationQuery={{ credential_store_validation_query }} + +########################################################################### +# AMQP Notification Configuration +########################################################################### +rabbitmq.broker.url={{ rabbitmq_broker_url }} +experiment.launch.queue={{ experiment_launch_queue }} +rabbitmq.status.exchange.name={{ rabbitmq_status_exchange_name }} +rabbitmq.process.exchange.name={{ rabbitmq_process_exchange_name }} +rabbitmq.experiment.exchange.name={{ rabbitmq_experiment_exchange_name }} +durable.queue={{ durable_queue }} +prefetch.count={{ prefetch_count }} + +########################################################################### +# Zookeeper Server Configuration +########################################################################### +embedded.zk={{ embedded_zk }} +zookeeper.server.connection={{ zookeeper_connection }} + +######################################################################## +## API Security Configuration +######################################################################## +security.manager.class={{ security_manager_class }} +TLS.enabled={{ TLS_enabled }} +TLS.client.timeout={{ TLS_client_timeout }} +keystore.path={{ keystore_path }} +keystore.password={{ keystore_password }} +authz.cache.enabled={{ authz_cache_enabled }} +authz.cache.manager.class={{ authz_cache_manager_class }} +in.memory.cache.size={{ in_memory_cache_size }} + +########################################################################### +# Profile Service Configuration +########################################################################### +profile.service.server.host={{ profile_service_host }} +profile.service.server.port={{ profile_service_port }} +profile_service.class={{ profile_service_class }} +# MariaDB properties +profile.service.jdbc.url={{ profile_service_jdbc_url }} +profile.service.jdbc.user={{ profile_service_jdbc_user }} +profile.service.jdbc.password={{ profile_service_jdbc_password }} +profile.service.jdbc.driver={{ profile_service_jdbc_driver }} +profile.service.validationQuery={{ profile_service_validation_query }} + +########################################################################### +# Iam Admin services Configuration +########################################################################### +iam.server.url={{ iam_server_url }} +iam.server.super.admin.username={{ iam_admin_username }} +iam.server.super.admin.password={{ iam_admin_password }} + +########################################################################### +# DB Event Manager Runner +########################################################################### +db_event_manager.class={{ db_event_manager_class }} + +########################################################################### +# Job Execution Engine properties +########################################################################### +helix.cluster.name={{ helix_cluster_name }} +helix.controller.name={{ helix_controller_name }} +helix.participant.name={{ helix_participant_name }} +participant.monitoring.enabled={{ participant_monitoring_enabled }} +participant.monitoring.host={{ participant_monitoring_host }} +participant.monitoring.port={{ participant_monitoring_port }} + +########################################################################### +# Job Monitor related properties +########################################################################### +enable.realtime.monitor={{ enable_realtime_monitor }} +realtime.monitor.broker.consumer.group={{ realtime_monitor_broker_consumer_group }} +realtime.monitor.broker.topic={{ realtime_monitor_broker_topic }} + +job.monitor.broker.consumer.group={{ job_monitor_broker_consumer_group }} +job.monitor.broker.topic={{ job_monitor_broker_topic }} +job.monitor.broker.publisher.id={{ job_monitor_broker_publisher_id }} +job.monitor.email.publisher.id={{ job_monitor_email_publisher_id }} +job.monitor.realtime.publisher.id={{ job_monitor_realtime_publisher_id }} + +email.based.monitor.host={{ email_based_monitor_host }} +email.based.monitor.store.protocol={{ email_based_monitor_store_protocol }} +email.based.monitor.folder.name={{ email_based_monitor_folder_name }} +email.expiration.minutes={{ email_expiration_minutes }} +email.based.monitoring.period={{ email_based_monitoring_period }} +email.based.monitor.address={{ email_based_monitor_address }} +email.based.monitor.password={{ email_based_monitor_password }} + +kafka.broker.url={{ kafka_broker_url }} +local.data.location={{ local_data_location }} + +########################################################################### +# ThriftClientPool Configuration +########################################################################### +thrift.client.pool.abandoned.removal.enabled={{ thrift_client_pool_abandoned_removal_enabled }} +thrift.client.pool.abandoned.removal.logged={{ thrift_client_pool_abandoned_removal_logged }} + +########################################################################### +# Pre-workflow Configuration +########################################################################### +pre.workflow.manager.loadbalance.clusters={{ pre_workflow_manager_loadbalance_clusters }} +pre.workflow.manager.monitoring.enabled={{ pre_workflow_manager_monitoring_enabled }} +pre.workflow.manager.monitoring.host={{ pre_workflow_manager_monitoring_host }} +pre.workflow.manager.monitoring.port={{ pre_wm_monitoring_port }} +pre.workflow.manager.name={{ pre_workflow_manager_name }} + +########################################################################### +# Post-workflow Configuration +########################################################################### +post.workflow.manager.loadbalance.clusters={{ post_workflow_manager_loadbalance_clusters }} +post.workflow.manager.monitoring.enabled={{ post_workflow_manager_monitoring_enabled }} +post.workflow.manager.monitoring.host={{ post_workflow_manager_monitoring_host }} +post.workflow.manager.monitoring.port={{ post_wm_monitoring_port }} +post.workflow.manager.name={{ post_workflow_manager_name }} + +########################################################################### +# Parser-workflow Configuration +########################################################################### +data.parser.delete.container={{ data_parser_delete_container }} +data.parser.broker.consumer.group={{ data_parser_broker_consumer_group }} +data.parser.topic={{ data_parser_topic }} +data.parser.storage.resource.id={{ data_parser_storage_resource_id }} + +########################################################################### +# Metascheduler And Compute Resource Monitoring Configuration +########################################################################### +cluster.status.monitoring.enable={{ cluster_status_monitoring_enable }} +# cluster.status.monitoring.repeat.time=18000 +metaschedluer.job.scanning.enable={{ metaschedluer_job_scanning_enable }} +data.analyzer.job.scanning.enable={{ data_analyzer_job_scanning_enable }} + +########################################################################### +# Data Staging Task Level Configurations +########################################################################### +enable.streaming.transfer={{ enable_streaming_transfer }} + diff --git a/dev-tools/ansible/roles/airavata_services/templates/application-agent-service.yml.j2 b/dev-tools/ansible/roles/airavata_services/templates/application-agent-service.yml.j2 new file mode 100644 index 0000000000..62295600d4 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/application-agent-service.yml.j2 @@ -0,0 +1,63 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +grpc: + server: + host: {{ agent_service_grpc_host }} + port: {{ agent_service_grpc_port }} + max-inbound-message-size: {{ agent_service_grpc_max_inbound_message_size }} + +server: + port: {{ agent_service_server_port }} + address: {{ agent_service_server_address }} + +spring: + servlet: + multipart: + max-file-size: {{ agent_service_max_file_size }} + max-request-size: {{ agent_service_max_request_size }} + datasource: + url: "{{ agent_service_datasource_url }}" + username: "{{ agent_service_datasource_username }}" + password: "{{ agent_service_datasource_password }}" + driver-class-name: org.mariadb.jdbc.Driver + hikari: + pool-name: {{ agent_service_pool_name }} + leak-detection-threshold: {{ agent_service_leak_detection_threshold }} + jpa: + hibernate: + ddl-auto: {{ agent_service_ddl_auto }} + open-in-view: false + +airavata: + server: + url: {{ agent_service_airavata_url }} + port: {{ agent_service_airavata_port }} + secure: {{ agent_service_airavata_secure }} + storageResourceId: {{ agent_service_storage_resource_id }} + storagePath: {{ agent_service_storage_path }} + cluster: + applicationInterfaceId: {{ agent_service_application_interface_id }} + tunnel: + serverHost: {{ tunnel_server_host }} + serverPort: {{ tunnel_server_port }} + serverToken: {{ tunnel_server_token }} + serverApiUrl: {{ tunnel_server_api_url }} + diff --git a/dev-tools/ansible/roles/airavata_services/templates/application-file-server.properties.j2 b/dev-tools/ansible/roles/airavata_services/templates/application-file-server.properties.j2 new file mode 100644 index 0000000000..4a8e80afc7 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/application-file-server.properties.j2 @@ -0,0 +1,34 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +spring.servlet.multipart.max-file-size={{ file_server_max_file_size }} +spring.servlet.multipart.max-request-size={{ file_server_max_request_size }} +spring.servlet.multipart.file-size-threshold={{ file_server_file_size_threshold }} + +# files storage location (stores all files uploaded via REST API) +storage.location={{ file_server_storage_location }} + +regserver.server.host={{ regserver_server_host }} +regserver.server.port={{ registry_server_port }} +credential.store.server.host={{ credential_store_server_host }} +credential.store.server.port={{ cred_store_port }} + +server.port={{ file_server_port }} + diff --git a/dev-tools/ansible/roles/airavata_services/templates/application-research-service.yml.j2 b/dev-tools/ansible/roles/airavata_services/templates/application-research-service.yml.j2 new file mode 100644 index 0000000000..b433c6027d --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/application-research-service.yml.j2 @@ -0,0 +1,76 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +grpc: + server: + port: {{ research_service_grpc_port }} + +server: + port: {{ research_service_server_port }} + address: {{ research_service_server_address }} + +airavata: + research-hub: + url: {{ research_hub_url }} + dev-user: "{{ research_hub_dev_user }}" + adminApiKey: "{{ research_hub_admin_api_key }}" + limit: {{ research_hub_limit }} + research-portal: + dev-url: {{ research_portal_dev_url }} + url: {{ research_portal_url }} + + openid: + url: "{{ openid_url }}" + + user-profile: + server: + url: {{ user_profile_server_url }} + port: {{ user_profile_server_port }} + +spring: + servlet: + multipart: + max-file-size: {{ research_service_max_file_size }} + max-request-size: {{ research_service_max_request_size }} + datasource: + url: "{{ research_service_datasource_url }}" + username: "{{ research_service_datasource_username }}" + password: "{{ research_service_datasource_password }}" + driver-class-name: org.mariadb.jdbc.Driver + hikari: + pool-name: {{ research_service_pool_name }} + leak-detection-threshold: {{ research_service_leak_detection_threshold }} + jpa: + hibernate: + ddl-auto: {{ research_service_ddl_auto }} + open-in-view: false + +springdoc: + api-docs: + enabled: {{ springdoc_api_docs_enabled }} + swagger-ui: + path: {{ springdoc_swagger_ui_path }} + operationsSorter: {{ springdoc_swagger_ui_operations_sorter }} + tagsSorter: {{ springdoc_swagger_ui_tags_sorter }} + doc-expansion: {{ springdoc_swagger_ui_doc_expansion }} + oauth: + use-pkce-with-authorization-code-grant: {{ springdoc_swagger_ui_oauth_use_pkce }} + client-id: {{ springdoc_swagger_ui_oauth_client_id }} + diff --git a/dev-tools/ansible/roles/airavata_services/templates/application-restproxy.properties.j2 b/dev-tools/ansible/roles/airavata_services/templates/application-restproxy.properties.j2 new file mode 100644 index 0000000000..0659c934e1 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/application-restproxy.properties.j2 @@ -0,0 +1,23 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +restproxy.broker.url={{ restproxy_broker_url }} +server.port={{ restproxy_port }} + diff --git a/dev-tools/ansible/roles/airavata_services/templates/email-config.yml.j2 b/dev-tools/ansible/roles/airavata_services/templates/email-config.yml.j2 new file mode 100644 index 0000000000..e361cfd320 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/email-config.yml.j2 @@ -0,0 +1,114 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +config: + resources: + - jobManagerType: PBS + emailParser: org.apache.airavata.monitor.email.parser.PBSEmailParser + resourceEmailAddresses: + - [email protected] # gordon + - [email protected] # Bigred2 + - root <[email protected]> # Bigred2 + - root <[email protected]> # alamo + - root <[email protected]> #karst + - root <[email protected]> #mason + - smic3 <[email protected]> # philip.hpc.lsu + - [email protected] + - adm <[email protected]> #supermic + - root <[email protected]> #alamo + - root <[email protected]> #carbonate + - adm <[email protected]> # Mick cluster at LSU for CSBG-LSU + - GW77 Job Emails <[email protected]> #gw77 email to fix email issues + + - jobManagerType: SLURM + emailParser: org.apache.airavata.monitor.email.parser.SLURMEmailParser + resourceEmailAddresses: + - SDSC Admin <[email protected]> # comet + - [email protected] # stampede + - SDSC Admin <[email protected]> # comet new + - [email protected] + - Slurm <[email protected]> # bridges + - Slurm <[email protected]> #Bridges + - Slurm Daemon <[email protected]> # OU Schooner + - [email protected] # stampede2 + - Slurm service account <[email protected]> # Utah Ember + - SLURM workload manager <[email protected]> # JS Mark Cluster + - super user <[email protected]> # LS5 new email after Nov 5th 2018 + - [email protected] #LS5 old email + - [email protected] # Jureca Email + - Slurm <[email protected]> + - [email protected] + - [email protected] + - [email protected] # Jetstream Ultrascan static cluster with 10 nodes + - [email protected] #Stampede2 + - [email protected] #Searching SRA + - Slurm Admin <[email protected]> #GSU cluster + - [email protected] #USD cluster + - [email protected] + - [email protected] + - SLURM resource manager <[email protected]> #Bigdawg + - SLURM resource manager <[email protected]> #InterACTWEL Jetstream + - slurm@zoar #R System cluster + - GW77 Job Emails <[email protected]> #gw77 email to fix email issues + - SLURM resource manager <[email protected]> #seagrid elastic cluster + - SLURM resource manager <[email protected]> # EPW Jetstream slurm cluster + - SLURM resource manager <[email protected]> #Distant reader + - [email protected] #RT at SDSU + - Slurm on Mio <[email protected]> # MIO from Mines + - SLURM resource manager <[email protected]> # Jetstream Helix cluster + - SLURM resource manager <[email protected]> + - [email protected] + - super user <root@localhost> # Bigred3 + - SLURM resource manager <[email protected]> + - SLURM resource manager <[email protected]> + - Slurm <[email protected]> + - [email protected] #Pinnacle UCMerced Cluster + - [email protected] #HIVE Slurm + - [email protected] #VLab PlasmaScience JS2 cluster + - [email protected] #Anvil Purdue + - SLURM User <[email protected]> + - [email protected] #Frontera + + - jobManagerType: UGE + emailParser: org.apache.airavata.monitor.email.parser.UGEEmailParser + resourceEmailAddresses: + - ls4.tacc.utexas.edu # contain Lonestar + - root <root@local> # USD HPC Cluster + - root <[email protected]> # SIU Little Dog + - [email protected] # SIU Big Dog + - root <[email protected]> # USD HPC Cluster + + - jobManagerType: LSF + emailParser: org.apache.airavata.monitor.email.parser.LSFEmailParser + resourceEmailAddresses: + - iu.xsede.edu # test resource mail address + - tcs.tulsahpc.org #Tandy + + - jobManagerType: HTCONDOR + emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser + resourceEmailAddresses: + - [email protected] + - Owner of HTCondor Daemons <[email protected]> #EHT Condor Access point + - Owner of HTCondor Daemons <[email protected]> + - [email protected] # AutoDock_Vina +# - jobManagerType: HTCondor +# emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser +# resourceEmailAddresses: + diff --git a/dev-tools/ansible/roles/airavata_services/templates/log4j2.xml.j2 b/dev-tools/ansible/roles/airavata_services/templates/log4j2.xml.j2 new file mode 100644 index 0000000000..3c8e581507 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/log4j2.xml.j2 @@ -0,0 +1,55 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--> +<Configuration status="WARN"> + <Properties> + <Property name="serviceName">${env:SERVICE_NAME:-output}</Property> + </Properties> + <Appenders> + <Console name="Console" target="SYSTEM_OUT"> + <PatternLayout pattern="%d [%t] %-5p %c{30} %X - %m%n"/> + </Console> + <RollingFile name="RollingFileAppender" fileName="../logs/${serviceName}.log" + filePattern="../logs/${serviceName}-%d{MM-dd-yyyy}-%i.log.gz"> + <PatternLayout> + <Pattern>%d [%t] %-5p %c{30} %X - %m%n</Pattern> + </PatternLayout> + <Policies> + <OnStartupTriggeringPolicy/> + <TimeBasedTriggeringPolicy/> + <SizeBasedTriggeringPolicy size="50 MB"/> + </Policies> + <DefaultRolloverStrategy max="20"/> + </RollingFile> + </Appenders> + <Loggers> + <logger name="ch.qos.logback" level="WARN"/> + <logger name="org.apache.helix" level="WARN"/> + <logger name="org.apache.zookeeper" level="ERROR"/> + <logger name="org.apache.airavata" level="INFO"/> + <logger name="org.hibernate" level="ERROR"/> + <Root level="INFO"> + <AppenderRef ref="Console"/> + <AppenderRef ref="RollingFileAppender"/> + </Root> + </Loggers> +</Configuration> +
