This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a commit to branch agent-framewok-refactoring
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/agent-framewok-refactoring by
this push:
new da5a94dd38 run an agent and jl locally for development
da5a94dd38 is described below
commit da5a94dd38bb7c9be7a815dd4917cf3b129f5c33
Author: lahiruj <[email protected]>
AuthorDate: Mon Oct 7 12:02:04 2024 -0400
run an agent and jl locally for development
---
.../jupyter/deployments/cerebrum/README.md | 24 +++
.../deployments/cerebrum/v1l4/docker-agent.sh | 10 +
.../cerebrum/v1l4/jupyterlab/Dockerfile | 26 +++
.../v1l4/jupyterlab/build-jupyter-image.sh | 1 +
.../cerebrum/v1l4/jupyterlab/labconfig/__init__.py | 0
.../v1l4/jupyterlab/labconfig/airavata_magics.py | 235 +++++++++++++++++++++
.../v1l4/jupyterlab/labconfig/bootstrap.sh | 2 +
.../jupyterlab/labconfig/jupyter_lab_config.py | 25 +++
.../v1l4/jupyterlab/run-jupyter-lab-container.sh | 1 +
9 files changed, 324 insertions(+)
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/README.md
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/README.md
new file mode 100644
index 0000000000..54f9fd41f7
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/README.md
@@ -0,0 +1,24 @@
+## Setting up Dev Environment
+
+### Agent
+Build the Agent
+```shell
+cd v1l4
+./build-container.sh
+```
+
+Execute the following command to run an agent locally.
+```shell
+docker run --rm -v $(pwd):/workspace -w /workspace
lahiruj/airavata-cerebrum-agent bash -c "./docker-agent.sh"
+```
+
+### Jupyter lab
+To run the jupyter lab
+
+```shell
+cd jupyterlab
+./build-jupyter-image.sh
+./run-jupyter-lab-container
+```
+
+
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/docker-agent.sh
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/docker-agent.sh
new file mode 100755
index 0000000000..924730aeca
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/docker-agent.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+if [ ! -d "/tmp/v1l4" ]; then
+ mkdir -p /tmp/v1l4
+fi
+
+cp -r /opt/v1l4/* /tmp/v1l4/
+
+/opt/airavata-agent scigap02.sciencegateways.iu.edu:19900 agent2
+
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/Dockerfile
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/Dockerfile
new file mode 100644
index 0000000000..f738f8495d
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/Dockerfile
@@ -0,0 +1,26 @@
+FROM python:slim
+
+USER root
+
+RUN apt update
+RUN apt install fuse -y
+RUN pip install notebook
+RUN apt install kmod -y
+RUN apt install git -y
+RUN pip install PyJWT
+RUN pip install cybershuttle-tune==0.1.19
+RUN pip install scipy
+RUN pip install numpy
+RUN pip install matplotlib
+RUN git clone https://github.com/cyber-shuttle/jupyter-notebook-examples
/home/jupyter-notebook-examples
+
+COPY labconfig/jupyter_lab_config.py /jupyter_lab_config.py
+COPY labconfig/airavata_magics.py /airavata_magics.py
+COPY labconfig/__init__.py /__init__.py
+COPY labconfig/bootstrap.sh /bootstrap.sh
+RUN chmod +x /bootstrap.sh
+
+EXPOSE 8888
+WORKDIR /home
+
+ENTRYPOINT ["sh", "/bootstrap.sh"]
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/build-jupyter-image.sh
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/build-jupyter-image.sh
new file mode 100755
index 0000000000..d6df5ac140
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/build-jupyter-image.sh
@@ -0,0 +1 @@
+docker build --platform linux/x86_64 -t lahiruj/cerebrum-jupyter-lab .
\ No newline at end of file
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/__init__.py
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/airavata_magics.py
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/airavata_magics.py
new file mode 100644
index 0000000000..cec1b2a10f
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/airavata_magics.py
@@ -0,0 +1,235 @@
+from IPython.core.magic import register_cell_magic
+from IPython.core.magic import register_line_magic
+
+from IPython.display import display, Image
+import base64
+import requests
+import json
+import jwt
+import time
+from pathlib import Path
+import os
+
+current_agent_info = {}
+current_agent_info["agentId"] = "agent2"
+
+EXPLICIT_TOKEN_FILE = (
+ Path(os.path.expanduser("~")) / "csagent" / "token" / "keys.json"
+)
+
+def read_explicit_token_file():
+ if not EXPLICIT_TOKEN_FILE.exists():
+ return None
+ else:
+ with open(EXPLICIT_TOKEN_FILE, "r") as f:
+ return json.load(f)
+
+def get_access_token():
+ expl_token_data = read_explicit_token_file()
+ if expl_token_data:
+ return expl_token_data["access_token"]
+
+def get_agent_status():
+ global current_agent_info
+
+ if not current_agent_info:
+ print("No agent was scheduled yet. Please run %init_remote
cluster=<cluster> cpu=<cpu> memory=<memory mb> queue=<queue> walltime=<walltime
minutes>")
+ return
+
+ url = 'http://scigap02.sciencegateways.iu.edu:18880/api/v1/agent/' +
current_agent_info['agentId']
+ response = requests.get(url)
+ if response.status_code == 202:
+ return response.json()
+ else:
+ print('Invalid response reveived. Status code:', response.status_code)
+ print('Response:', response.text)
+
+def submit_agent_job(experiment_name, cluster, queue, cpus, memory, wallTime,
access_token, gateway_id='testdrive'):
+
+ global current_agent_info
+ # URL to which the POST request will be sent
+ url = 'http://scigap02.sciencegateways.iu.edu:18880/api/v1/exp/launch'
+
+ # Data to be sent in the POST request
+ data = {
+ 'experimentName': experiment_name,
+ 'remoteCluster': cluster,
+ 'cpuCount': cpus,
+ 'nodeCount': 1,
+ 'memory': memory,
+ 'wallTime': wallTime,
+ 'queue': queue
+ }
+
+ # Convert the data to JSON format
+ json_data = json.dumps(data)
+
+ decode = jwt.decode(access_token, options={"verify_signature": False})
+ user_id = decode['preferred_username']
+ claimsMap = {
+ "userName": user_id,
+ "gatewayID": gateway_id
+ }
+
+ # Headers
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Authorization': 'Bearer ' + access_token,
+ 'X-Claims': json.dumps(claimsMap)
+ }
+
+ # Send the POST request
+ response = requests.post(url, headers=headers, data=json_data)
+
+ # Check if the request was successful
+ if response.status_code == 200:
+ # Parse the JSON response
+ response_data = response.json()
+ print('Response:', response_data)
+ current_agent_info = response_data
+ else:
+ print('Failed to send POST request. Status code:',
response.status_code)
+ print('Response:', response.text)
+
+def terminate_agent(access_token, gateway_id='testdrive'):
+
+ global current_agent_info
+
+ expId = current_agent_info['experimentId']
+ url = 'http://scigap02.sciencegateways.iu.edu:18880/api/v1/exp/terminate/'
+ expId
+
+ decode = jwt.decode(access_token, options={"verify_signature": False})
+ user_id = decode['preferred_username']
+ claimsMap = {
+ "userName": user_id,
+ "gatewayID": gateway_id
+ }
+
+ # Headers
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Authorization': 'Bearer ' + access_token,
+ 'X-Claims': json.dumps(claimsMap)
+ }
+
+ # Send the POST request
+ response = requests.get(url, headers=headers)
+
+ if response.status_code == 200:
+ # Parse the JSON response
+ response_data = response.json()
+ print('Agent terminated:', response_data)
+ current_agent_info = None
+ else:
+ print('Failed to send termination request. Status code:',
response.status_code)
+ print('Response:', response.text)
+
+
+@register_cell_magic
+def run_remote(line, cell):
+
+ global current_agent_info
+ if not current_agent_info:
+ print("No agent was scheduled yet. Please run %init_remote
cluster=<cluster> cpu=<cpu> memory=<memory mb> queue=<queue> walltime=<walltime
minutes>")
+
+ url =
'http://scigap02.sciencegateways.iu.edu:18880/api/v1/agent/executejupyterrequest'
+
+ data = {
+ "sessionId": "session1",
+ "keepAlive": True,
+ "code": cell,
+ "agentId": current_agent_info["agentId"]
+ }
+
+ json_data = json.dumps(data)
+ response = requests.post(url, headers={'Content-Type':
'application/json'}, data=json_data)
+ execution_resp = response.json()
+ execution_id = execution_resp["executionId"]
+ error = execution_resp["error"]
+ if error:
+ print("Cell execution failed. Error: " + error)
+ if execution_id:
+ while True:
+ url =
"http://scigap02.sciencegateways.iu.edu:18880/api/v1/agent/executejupyterresponse/";
+ execution_id
+ response = requests.get(url, headers={'Accept':
'application/json'})
+ json_response = response.json()
+ #print(json_response)
+ if json_response['available']:
+ result_str = json_response['responseString']
+ result = json.loads(result_str)
+ if 'result' in result:
+ print(result['result'])
+ elif 'error' in result:
+ print(result['error']['ename'])
+ print(result['error']['evalue'])
+ print(result['error']['traceback'])
+ elif 'display' in result:
+ display_obj = result['display']
+ if 'data' in display_obj:
+ data_obj = display_obj['data']
+ if 'image/png' in data_obj:
+ image_data = data_obj['image/png']
+ display(Image(data=base64.b64decode(image_data),
format='png'))
+ break
+ time.sleep(1)
+
+@register_line_magic
+def init_remote(line):
+
+ if current_agent_info:
+ status = get_agent_status()
+ if status:
+ if status['agentUp']:
+ print("An agent is already running. Please terminate it first
by running %terminate_remote")
+ return
+ else:
+ print("An agent was scheduled. Please terminate it first by
running %terminate_remote")
+ return
+
+ access_token = get_access_token()
+ pairs = line.split()
+
+ # Initialize variable to store the cluster value
+ cluster_value = None
+ memory_value = None
+ cpu_value = None
+ queue_value = None
+ walltime_value = None
+
+ # Iterate through the pairs to find the cluster value
+ for pair in pairs:
+ if pair.startswith("cluster="):
+ cluster_value = pair.split("=")[1]
+ if pair.startswith("cpu="):
+ cpu_value = pair.split("=")[1]
+ if pair.startswith("memory="):
+ memory_value = pair.split("=")[1]
+ if pair.startswith("queue="):
+ queue_value = pair.split("=")[1]
+ if pair.startswith("walltime="):
+ walltime_value = pair.split("=")[1]
+
+ submit_agent_job('CS Agent', cluster_value, queue_value, cpu_value,
memory_value, walltime_value, access_token)
+
+@register_line_magic
+def status_remote(line):
+ status = get_agent_status()
+ if status:
+ if status['agentUp']:
+ print("Agent", status['agentId'], 'is running')
+ else:
+ print("Agent", status['agentId'], 'is still prepairing. Please
wait')
+
+@register_line_magic
+def terminate_remote(line):
+ global current_agent_info
+ access_token = get_access_token()
+ if current_agent_info:
+ terminate_agent(access_token)
+
+
+def load_ipython_extension(ipython):
+ ipython.register_magic_function(init_remote)
+ ipython.register_magic_function(status_remote)
+ ipython.register_magic_function(terminate_remote)
+ ipython.register_magic_function(run_remote)
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/bootstrap.sh
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/bootstrap.sh
new file mode 100644
index 0000000000..754a6e8f52
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/bootstrap.sh
@@ -0,0 +1,2 @@
+cd /home/jupyter-notebook-examples && git pull && cd -
+jupyter lab --config=/jupyter_lab_config.py --ip=0.0.0.0 --port=8888
--no-browser --allow-root --NotebookApp.token=''
\ No newline at end of file
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/jupyter_lab_config.py
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/jupyter_lab_config.py
new file mode 100644
index 0000000000..a160cd77cb
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/labconfig/jupyter_lab_config.py
@@ -0,0 +1,25 @@
+import sys
+sys.path.append('/')
+
+c = get_config()
+
+c.InteractiveShellApp.exec_lines = [
+ "import sys"
+ "sys.path.append('/')"
+ "import airavata_magics",
+ "airavata_magics.load_ipython_extension(get_ipython())"
+]
+
+# Set the IP address Jupyter Lab will listen on
+c.ServerApp.ip = '0.0.0.0'
+
+# Set the port Jupyter Lab will listen on
+c.ServerApp.port = 8888
+
+# Don't open the browser by default
+c.ServerApp.open_browser = False
+
+c.FileContentsManager.use_atomic_writing = False
+
+# Allow root access
+c.ServerApp.allow_root = True
\ No newline at end of file
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/run-jupyter-lab-container.sh
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/run-jupyter-lab-container.sh
new file mode 100755
index 0000000000..59906b0c8d
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/cerebrum/v1l4/jupyterlab/run-jupyter-lab-container.sh
@@ -0,0 +1 @@
+docker run -p 18888:8888 -it lahiruj/cerebrum-jupyter-lab
\ No newline at end of file
