This is an automated email from the ASF dual-hosted git repository.
yasith pushed a commit to branch cybershuttle-staging
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/cybershuttle-staging by this
push:
new ee2ca801d0 add separate --environ input to agent. add js2 targets to
makefile. update sdk to pass environ
ee2ca801d0 is described below
commit ee2ca801d029fd9901aa6129f625027ee2b411f8
Author: yasithdev <[email protected]>
AuthorDate: Thu Apr 10 15:09:37 2025 -0400
add separate --environ input to agent. add js2 targets to makefile. update
sdk to pass environ
---
.../airavata_jupyter_magic/__init__.py | 16 ++++++++++++----
.../airavata-python-sdk/pyproject.toml | 2 +-
modules/agent-framework/airavata-agent/Makefile | 19 ++++++++++++++++++-
modules/agent-framework/airavata-agent/agent.go | 18 ++++++++++++++++--
modules/agent-framework/airavata-agent/agent.sh | 7 +++++--
5 files changed, 52 insertions(+), 10 deletions(-)
diff --git
a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_jupyter_magic/__init__.py
b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_jupyter_magic/__init__.py
index 827f7069e4..71bf0f1e00 100644
---
a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_jupyter_magic/__init__.py
+++
b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_jupyter_magic/__init__.py
@@ -1,5 +1,6 @@
import base64
import binascii
+import hashlib
import importlib.metadata
import json
import os
@@ -12,12 +13,11 @@ from typing import Any, NamedTuple, Optional
import jwt
import requests
-import tomli
import yaml
from IPython.core.getipython import get_ipython
from IPython.core.interactiveshell import ExecutionResult
from IPython.core.magic import register_cell_magic, register_line_magic
-from IPython.display import HTML, Image, display, Javascript, HTML
+from IPython.display import HTML, Image, Javascript, display
from rich.console import Console
from .device_auth import DeviceFlowAuthenticator
@@ -72,8 +72,14 @@ RuntimeInfo = NamedTuple('RuntimeInfo', [
('walltime', int),
('gateway_id', str),
('group', str),
+ ('libraries', list[str]),
+ ('pip', list[str]),
])
+def getEnvName(rt: RuntimeInfo) -> str:
+ string = ''.join(rt.libraries + rt.pip)
+ return hashlib.sha256(string.encode('utf-8')).hexdigest()
+
PENDING_STATES = [
ProcessState.CREATED,
@@ -355,6 +361,8 @@ def submit_agent_job(
memory=data['memory'],
walltime=data['wallTime'],
group=data['group'],
+ libraries=data['libraries'],
+ pip=data['pip'],
)
state.all_runtimes[rt_name] = rt
print(f'Requested runtime={rt_name}. state={pstate.name}')
@@ -420,7 +428,7 @@ def restart_runtime_kernel(access_token: str, rt_name: str,
env_name: str, runti
# Send the POST request
res = requests.post(url, headers=headers, data=json.dumps({
"agentId": runtime.agentId,
- "envName": env_name,
+ "envName": getEnvName(runtime),
}))
data = res.json()
@@ -490,7 +498,7 @@ def run_on_runtime(rt_name: str, code_obj: str, result:
ExecutionResult) -> bool
url = api_base_url + '/api/v1/agent/execute/jupyter'
data = {
"agentId": rt.agentId,
- "envName": rt.agentId,
+ "envName": getEnvName(rt),
"code": code_obj,
}
json_data = json.dumps(data)
diff --git
a/airavata-api/airavata-client-sdks/airavata-python-sdk/pyproject.toml
b/airavata-api/airavata-client-sdks/airavata-python-sdk/pyproject.toml
index 6f3a1e0044..3cb42553ff 100644
--- a/airavata-api/airavata-client-sdks/airavata-python-sdk/pyproject.toml
+++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "airavata-python-sdk"
-version = "2.0.9"
+version = "2.0.10"
description = "Apache Airavata Python SDK"
readme = "README.md"
license = "Apache-2.0"
diff --git a/modules/agent-framework/airavata-agent/Makefile
b/modules/agent-framework/airavata-agent/Makefile
index 6a6d2180cc..3bc78f03c7 100644
--- a/modules/agent-framework/airavata-agent/Makefile
+++ b/modules/agent-framework/airavata-agent/Makefile
@@ -49,9 +49,26 @@ deploy-expanse-gridchem:
cybershuttle/container/remote-agent-base.sif \
docker://$(IMAGE_NAME)"
+deploy-neurodata25-vc: deploy-neurodata25-vc-1 deploy-neurodata25-vc-2
+
+deploy-neurodata25-vc-1:
+ ssh [email protected] \
+ "module load singularity && srun -p cloud -t 30 \
+ singularity pull --disable-cache --force \
+ cybershuttle/container/remote-agent-base.sif \
+ docker://$(IMAGE_NAME)"
+
+deploy-neurodata25-vc-2:
+ ssh [email protected] \
+ "module load singularity && srun -p cloud -t 30 \
+ singularity pull --disable-cache --force \
+ cybershuttle/container/remote-agent-base.sif \
+ docker://$(IMAGE_NAME)"
+
deploy-bootstrap:
scp agent.sh x-scigap@anvil:~/cybershuttle/agent.sh && \
scp agent.sh x-waterhub@anvil:~/cybershuttle/agent.sh && \
scp agent.sh x-gcommunityus@anvil:~/cybershuttle/agent.sh && \
scp agent.sh scigap@expanse:~/cybershuttle/agent.sh && \
- scp agent.sh gridchem@expanse:~/cybershuttle/agent.sh
+ scp agent.sh [email protected]:~/cybershuttle/agent.sh && \
+ scp agent.sh [email protected]:~/cybershuttle/agent.sh
diff --git a/modules/agent-framework/airavata-agent/agent.go
b/modules/agent-framework/airavata-agent/agent.go
index 973ed9f510..822e35c681 100644
--- a/modules/agent-framework/airavata-agent/agent.go
+++ b/modules/agent-framework/airavata-agent/agent.go
@@ -33,6 +33,7 @@ func main() {
// Define flags with default empty values.
serverUrl := flag.String("server", "", "Server flag (optional)")
agentId := flag.String("agent", "", "Agent flag (optional)")
+ environ := flag.String("environ", "", "Environment name (optional)")
lib := flag.String("lib", "", "Libraries flag (optional)")
pip := flag.String("pip", "", "Pip flag (optional)")
@@ -40,6 +41,7 @@ func main() {
flag.Parse()
log.Printf("[agent.go] main() --server=%s\n", *serverUrl)
log.Printf("[agent.go] main() --agent=%s\n", *agentId)
+ log.Printf("[agent.go] main() --environ=%s\n", *environ)
log.Printf("[agent.go] main() --lib=%s\n", *lib)
log.Printf("[agent.go] main() --pip=%s\n", *pip)
@@ -50,6 +52,9 @@ func main() {
if *agentId == "" {
log.Fatalf("[agent.go] main() Error: --agent flag is
required.\n")
}
+ if *environ == "" {
+ log.Fatalf("[agent.go] main() Error: --environ flag is
required.\n")
+ }
ctx := context.Background()
conn, err := grpc.DialContext(
@@ -76,11 +81,20 @@ func main() {
}
log.Printf("[agent.go] main() Created stream...\n")
+ // create environment, don't recreate if exists
+ envCmd := exec.Command("micromamba", "create", "-n", *environ)
+ envCmd.Stdin = strings.NewReader("n\n")
+ if err := envCmd.Run(); err != nil {
+ log.Printf("[agent.go] main() Using environment: %s", *environ)
+ } else {
+ log.Printf("[agent.go] main() Created environment: %s\n",
*environ)
+ }
+
var libList []string
if strings.TrimSpace(*lib) != "" {
libList = append(strings.Split(*lib, ","), defaultLibs...)
log.Printf("[agent.go] main() Installing --lib: %v\n", libList)
- libCmd := exec.Command("micromamba", "install", "-n", *agentId,
"--yes")
+ libCmd := exec.Command("micromamba", "install", "-n", *environ,
"--yes")
libCmd.Args = append(libCmd.Args, libList...)
libCmd.Stdout = os.Stdout
libCmd.Stderr = os.Stderr
@@ -96,7 +110,7 @@ func main() {
if strings.TrimSpace(*pip) != "" {
pipList = strings.Split(*pip, ",")
log.Printf("[agent.go] main() Installing --pip: %v\n", pipList)
- pipCmd := exec.Command("micromamba", "run", "-n", *agentId,
"pip", "install")
+ pipCmd := exec.Command("micromamba", "run", "-n", *environ,
"pip", "install")
pipCmd.Args = append(pipCmd.Args, pipList...)
pipCmd.Stdout = os.Stdout
pipCmd.Stderr = os.Stderr
diff --git a/modules/agent-framework/airavata-agent/agent.sh
b/modules/agent-framework/airavata-agent/agent.sh
index cdc0a81197..c70d50caa2 100755
--- a/modules/agent-framework/airavata-agent/agent.sh
+++ b/modules/agent-framework/airavata-agent/agent.sh
@@ -22,10 +22,11 @@ AGENT=""
SERVER=""
CONTAINER=""
LIBRARIES=""
+ENVIRON=""
PIP=""
BIND_OPTS=()
-PARSED_OPTIONS=$(getopt -o '' --long
server:,agent:,container:,libraries:,pip:,mounts:,bind: -n "$0" -- "$@")
+PARSED_OPTIONS=$(getopt -o '' --long
server:,agent:,container:,libraries:,pip:,mounts:,environ:,bind: -n "$0" --
"$@")
if [ $? -ne 0 ]; then
echo "Usage: $0 \
--server SERVER \
@@ -34,6 +35,7 @@ if [ $? -ne 0 ]; then
--libraries LIBRARIES \
--pip PIP \
--mounts MOUNTS \
+ --environ ENVIRON
[--bind BIND] ..."
exit 1
fi
@@ -53,6 +55,7 @@ while true; do
BIND_OPTS+=("--bind $CS_HOME/dataset/$SRC:$DEST:ro")
done
shift 2 ;;
+ --environ) ENVIRON="$2"; shift 2 ;;
--bind) BIND_OPTS+=("--bind $2:ro"); shift 2 ;;
--) shift; break ;;
*) echo "Unexpected option: $1"; exit 1 ;;
@@ -80,4 +83,4 @@ singularity exec \
--env MAMBA_ROOT_PREFIX=/scratch \
--env TMPDIR=/scratch/tmp \
$CS_HOME/container/$CONTAINER \
- bash -c "micromamba create -n $AGENT && /opt/airavata-agent --server
\"$SERVER:19900\" --agent \"$AGENT\" --lib \"$LIBRARIES\" --pip \"$PIP\""
+ bash -c "/opt/airavata-agent --server \"$SERVER:19900\" --agent \"$AGENT\"
--environ \"$ENVIRON\" --lib \"$LIBRARIES\" --pip \"$PIP\""
