This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch agent-framewok-refactoring
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/agent-framewok-refactoring by
this push:
new 030e80aeb8 i-guide templates
030e80aeb8 is described below
commit 030e80aeb8ba02c07f6c62b56cfe73d6fe940781
Author: Dimuthu Wannipurage <[email protected]>
AuthorDate: Tue Sep 17 23:34:08 2024 -0400
i-guide templates
---
.../airavata-agent/jupyter/Dockerfile | 5 +-
.../airavata-agent/jupyter/build-jupyter-image.sh | 1 +
.../jupyter/deployments/i-guide/agent/Dockerfile | 20 +++++
.../deployments/i-guide/agent/build-container.sh | 6 ++
.../jupyter/deployments/i-guide/agent/kernel.py | 100 +++++++++++++++++++++
.../deployments/i-guide/jupyterlab/Dockerfile | 26 ++++++
.../i-guide/jupyterlab/build-jupyter-image.sh | 1 +
.../i-guide/jupyterlab/labconfig/__init__.py | 0
.../jupyterlab}/labconfig/airavata_magics.py | 13 +--
.../i-guide/jupyterlab/labconfig/bootstrap.sh | 2 +
.../jupyterlab/labconfig/jupyter_lab_config.py | 25 ++++++
.../jupyterlab/run-jupyter-lab-container.sh | 1 +
.../jupyter/labconfig/airavata_magics.py | 10 +--
13 files changed, 196 insertions(+), 14 deletions(-)
diff --git a/modules/agent-framework/airavata-agent/jupyter/Dockerfile
b/modules/agent-framework/airavata-agent/jupyter/Dockerfile
index a2bb2cf054..e0a42a7240 100644
--- a/modules/agent-framework/airavata-agent/jupyter/Dockerfile
+++ b/modules/agent-framework/airavata-agent/jupyter/Dockerfile
@@ -10,15 +10,14 @@ RUN pip install cybershuttle-tune==0.1.19
RUN pip install scipy
RUN pip install numpy
RUN pip install matplotlib
-
+RUN git clone https://github.com/cyber-shuttle/jupyter-notebook-examples
/home/jupyter-notebook-examples
COPY labconfig/jupyter_lab_config.py /jupyter_lab_config.py
COPY labconfig/airavata_magics.py /airavata_magics.py
COPY labconfig/__init__.py /__init__.py
-RUN git clone https://github.com/cyber-shuttle/jupyter-notebook-examples
/home/jupyter-notebook-examples
COPY labconfig/bootstrap.sh /bootstrap.sh
RUN chmod +x /bootstrap.sh
-COPY fuse/client /client
+# COPY fuse/client /client
EXPOSE 8888
WORKDIR /home
diff --git
a/modules/agent-framework/airavata-agent/jupyter/build-jupyter-image.sh
b/modules/agent-framework/airavata-agent/jupyter/build-jupyter-image.sh
new file mode 100755
index 0000000000..8f06f6b52c
--- /dev/null
+++ b/modules/agent-framework/airavata-agent/jupyter/build-jupyter-image.sh
@@ -0,0 +1 @@
+docker build --platform linux/x86_64 -t dimuthuupe/airavata-jupyter-lab .
\ No newline at end of file
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/agent/Dockerfile
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/agent/Dockerfile
new file mode 100644
index 0000000000..790e826d35
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/agent/Dockerfile
@@ -0,0 +1,20 @@
+FROM rkalyana/iguide-singleuser-cvmfs:nbfetch
+
+USER root
+
+RUN pip install pandas
+RUN pip install geopandas
+RUN pip install pygeos
+RUN pip install matplotlib
+RUN pip install "numpy<2"
+RUN apt update
+RUN apt install -y gdal-bin
+RUN pip install -U libpysal
+RUN pip install -U esda
+RUN pip3 install contextily
+
+RUN pip install flask jupyter jupyter-client
+RUN mkdir -p /opt/jupyter
+RUN python -m venv /opt/jupyter/venv
+ADD airavata-agent-linux /opt/airavata-agent
+ADD kernel.py /opt/jupyter/kernel.py
\ No newline at end of file
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/agent/build-container.sh
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/agent/build-container.sh
new file mode 100755
index 0000000000..a15af8d8ac
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/agent/build-container.sh
@@ -0,0 +1,6 @@
+cd ../../../../
+env GOOS=linux GOARCH=amd64 go build
+cp airavata-agent jupyter/deployments/i-guide/agent/airavata-agent-linux
+cd jupyter/deployments/i-guide/agent
+docker build --platform linux/x86_64 -t dimuthuupe/airavata-iguide-agent .
+docker push dimuthuupe/airavata-iguide-agent
\ No newline at end of file
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/agent/kernel.py
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/agent/kernel.py
new file mode 100644
index 0000000000..5188105732
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/agent/kernel.py
@@ -0,0 +1,100 @@
+import time
+from jupyter_client import KernelManager
+from flask import Flask, request, jsonify
+import os
+import json
+
+
+app = Flask(__name__)
+
+km = None
+kc = None
+
+kernel_running = False
+
[email protected]('/start', methods=['GET'])
+def start_kernel():
+
+ global km
+ global kc
+ global kernel_running
+
+ if kernel_running:
+ return "Kernel already running"
+ # Create a new kernel manager
+ km = KernelManager(kernel_name='python3')
+ km.start_kernel()
+
+ # Create a client to interact with the kernel
+ kc = km.client()
+ kc.start_channels()
+
+ # Ensure the client is connected before executing code
+ kc.wait_for_ready()
+ kernel_running = True
+ return "Kernel started"
+
[email protected]('/execute', methods=['POST'])
+def execute():
+
+ global km
+ global kc
+
+ code = request.json.get('code', '')
+ if not code:
+ return jsonify({'error': 'No code provided'}), 400
+
+ kc.execute(code)
+
+ # Wait for the result and display it
+
+ execution_noticed = False
+ content_text = ""
+ while True:
+ try:
+ msg = kc.get_iopub_msg(timeout=1)
+ print("------------------")
+ print(msg)
+ print("-================-")
+ content = msg["content"]
+ parent_header = msg["parent_header"]
+
+ # When a message with the text stream comes and it's the result of
our execution
+ if msg["msg_type"] == "execute_input":
+ execution_noticed = True
+ if msg["msg_type"] == "stream" and content["name"] == "stdout":
+ print(content["text"])
+ content_text = content_text + content["text"]
+ if msg["msg_type"] == "display_data":
+ return jsonify({'display': content}), 200
+ if msg["msg_type"] == "error":
+ return jsonify({'error': content}), 200
+ if msg["msg_type"] == "status" and execution_noticed:
+ if content["execution_state"] and content["execution_state"]
== "idle":
+ if parent_header and parent_header["msg_type"]:
+ if parent_header["msg_type"] == "execute_request": ##
This is a result without stdout like a = 12
+ return jsonify({'result': content_text}), 200
+ except KeyboardInterrupt:
+ print("Interrupted by user.")
+ return jsonify({'error': "Intterrupted by user"}), 500
+ except:
+ pass
+
[email protected]('/stop', methods=['GET'])
+def stop():
+
+ global km
+ global kc
+ global kernel_running
+
+ if not kernel_running:
+ return "Kernel is not running to shut down"
+
+ kc.stop_channels()
+ km.shutdown_kernel()
+ kernel_running = False
+ return 'Kernel shutting down...'
+
+
+if __name__ == '__main__':
+ app.run(port=15000)
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/Dockerfile
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/Dockerfile
new file mode 100644
index 0000000000..781a4e2133
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/Dockerfile
@@ -0,0 +1,26 @@
+FROM rkalyana/iguide-singleuser-cvmfs:nbfetch
+
+USER root
+
+RUN pip install pandas
+RUN pip install geopandas
+RUN pip install pygeos
+RUN pip install matplotlib
+RUN pip install "numpy<2"
+RUN apt update
+RUN apt install -y gdal-bin
+RUN pip install -U libpysal
+RUN pip install -U esda
+RUN pip3 install contextily
+
+COPY labconfig/jupyter_lab_config.py /jupyter_lab_config.py
+COPY labconfig/airavata_magics.py /airavata_magics.py
+COPY labconfig/__init__.py /__init__.py
+COPY labconfig/bootstrap.sh /bootstrap.sh
+RUN chmod +x /bootstrap.sh
+
+USER ${NB_UID}
+RUN git clone
https://github.com/DImuthuUpe/population_vulnerable_to_dam_failure
/home/jovyan/population_vulnerable_to_dam_failure
+# COPY fuse/client /client
+
+CMD ["start-notebook.sh","--NotebookApp.iopub_data_rate_limit=1e10"]
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/build-jupyter-image.sh
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/build-jupyter-image.sh
new file mode 100755
index 0000000000..cdf1966a11
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/build-jupyter-image.sh
@@ -0,0 +1 @@
+docker build --platform linux/x86_64 -t dimuthuupe/iguide-jupyter-lab .
\ No newline at end of file
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/__init__.py
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git
a/modules/agent-framework/airavata-agent/jupyter/labconfig/airavata_magics.py
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/airavata_magics.py
similarity index 93%
copy from
modules/agent-framework/airavata-agent/jupyter/labconfig/airavata_magics.py
copy to
modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/airavata_magics.py
index c3cf49c204..9f78698e34 100644
---
a/modules/agent-framework/airavata-agent/jupyter/labconfig/airavata_magics.py
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/airavata_magics.py
@@ -10,7 +10,8 @@ import time
from pathlib import Path
import os
-current_agent_info = None
+current_agent_info = {}
+current_agent_info["agentId"] = "agent2"
EXPLICIT_TOKEN_FILE = (
Path(os.path.expanduser("~")) / "csagent" / "token" / "keys.json"
@@ -35,7 +36,7 @@ def get_agent_status():
print("No agent was scheduled yet. Please run %init_remote
cluster=<cluster> cpu=<cpu> memory=<memory mb> queue=<queue> walltime=<walltime
minutes>")
return
- url = 'http://scigap02.sciencegateways.iu.edu:18880/api/v1/agent/' +
current_agent_info['agentId']
+ url = 'https://api.gateway.cybershuttle.org/api/v1/agent/' +
current_agent_info['agentId']
response = requests.get(url)
if response.status_code == 202:
return response.json()
@@ -47,7 +48,7 @@ def submit_agent_job(experiment_name, cluster, queue, cpus,
memory, wallTime, ac
global current_agent_info
# URL to which the POST request will be sent
- url = 'http://scigap02.sciencegateways.iu.edu:18880/api/v1/exp/launch'
+ url = 'https://api.gateway.cybershuttle.org/api/v1/exp/launch'
# Data to be sent in the POST request
data = {
@@ -95,7 +96,7 @@ def terminate_agent(access_token, gateway_id='testdrive'):
global current_agent_info
expId = current_agent_info['experimentId']
- url = 'http://scigap02.sciencegateways.iu.edu:18880/api/v1/exp/terminate/'
+ expId
+ url = 'https://api.gateway.cybershuttle.org/api/v1/exp/terminate/' + expId
decode = jwt.decode(access_token, options={"verify_signature": False})
user_id = decode['preferred_username']
@@ -131,7 +132,7 @@ def run_remote(line, cell):
if not current_agent_info:
print("No agent was scheduled yet. Please run %init_remote
cluster=<cluster> cpu=<cpu> memory=<memory mb> queue=<queue> walltime=<walltime
minutes>")
- url =
'http://scigap02.sciencegateways.iu.edu:18880/api/v1/agent/executejupyterrequest'
+ url =
'https://api.gateway.cybershuttle.org/api/v1/agent/executejupyterrequest'
data = {
"sessionId": "session1",
@@ -149,7 +150,7 @@ def run_remote(line, cell):
print("Cell execution failed. Error: " + error)
if execution_id:
while True:
- url =
"http://scigap02.sciencegateways.iu.edu:18880/api/v1/agent/executejupyterresponse/";
+ execution_id
+ url =
"https://api.gateway.cybershuttle.org/api/v1/agent/executejupyterresponse/"; +
execution_id
response = requests.get(url, headers={'Accept':
'application/json'})
json_response = response.json()
#print(json_response)
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/bootstrap.sh
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/bootstrap.sh
new file mode 100644
index 0000000000..8c163678b7
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/bootstrap.sh
@@ -0,0 +1,2 @@
+cd /home/population_vulnerable_to_dam_failure && git pull && cd -
+jupyter lab --config=/jupyter_lab_config.py --ip=0.0.0.0 --port=8888
--no-browser --allow-root --NotebookApp.token=''
\ No newline at end of file
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/jupyter_lab_config.py
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/jupyter_lab_config.py
new file mode 100644
index 0000000000..a160cd77cb
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/labconfig/jupyter_lab_config.py
@@ -0,0 +1,25 @@
+import sys
+sys.path.append('/')
+
+c = get_config()
+
+c.InteractiveShellApp.exec_lines = [
+ "import sys"
+ "sys.path.append('/')"
+ "import airavata_magics",
+ "airavata_magics.load_ipython_extension(get_ipython())"
+]
+
+# Set the IP address Jupyter Lab will listen on
+c.ServerApp.ip = '0.0.0.0'
+
+# Set the port Jupyter Lab will listen on
+c.ServerApp.port = 8888
+
+# Don't open the browser by default
+c.ServerApp.open_browser = False
+
+c.FileContentsManager.use_atomic_writing = False
+
+# Allow root access
+c.ServerApp.allow_root = True
\ No newline at end of file
diff --git
a/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/run-jupyter-lab-container.sh
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/run-jupyter-lab-container.sh
new file mode 100755
index 0000000000..ebc2d547f6
--- /dev/null
+++
b/modules/agent-framework/airavata-agent/jupyter/deployments/i-guide/jupyterlab/run-jupyter-lab-container.sh
@@ -0,0 +1 @@
+docker run -p 18888:8888 -it dimuthuupe/iguide-jupyter-lab
\ No newline at end of file
diff --git
a/modules/agent-framework/airavata-agent/jupyter/labconfig/airavata_magics.py
b/modules/agent-framework/airavata-agent/jupyter/labconfig/airavata_magics.py
index c3cf49c204..511cb9e671 100644
---
a/modules/agent-framework/airavata-agent/jupyter/labconfig/airavata_magics.py
+++
b/modules/agent-framework/airavata-agent/jupyter/labconfig/airavata_magics.py
@@ -35,7 +35,7 @@ def get_agent_status():
print("No agent was scheduled yet. Please run %init_remote
cluster=<cluster> cpu=<cpu> memory=<memory mb> queue=<queue> walltime=<walltime
minutes>")
return
- url = 'http://scigap02.sciencegateways.iu.edu:18880/api/v1/agent/' +
current_agent_info['agentId']
+ url = 'https://api.gateway.cybershuttle.org/api/v1/agent/' +
current_agent_info['agentId']
response = requests.get(url)
if response.status_code == 202:
return response.json()
@@ -47,7 +47,7 @@ def submit_agent_job(experiment_name, cluster, queue, cpus,
memory, wallTime, ac
global current_agent_info
# URL to which the POST request will be sent
- url = 'http://scigap02.sciencegateways.iu.edu:18880/api/v1/exp/launch'
+ url = 'https://api.gateway.cybershuttle.org/api/v1/exp/launch'
# Data to be sent in the POST request
data = {
@@ -95,7 +95,7 @@ def terminate_agent(access_token, gateway_id='testdrive'):
global current_agent_info
expId = current_agent_info['experimentId']
- url = 'http://scigap02.sciencegateways.iu.edu:18880/api/v1/exp/terminate/'
+ expId
+ url = 'https://api.gateway.cybershuttle.org/api/v1/exp/terminate/' + expId
decode = jwt.decode(access_token, options={"verify_signature": False})
user_id = decode['preferred_username']
@@ -131,7 +131,7 @@ def run_remote(line, cell):
if not current_agent_info:
print("No agent was scheduled yet. Please run %init_remote
cluster=<cluster> cpu=<cpu> memory=<memory mb> queue=<queue> walltime=<walltime
minutes>")
- url =
'http://scigap02.sciencegateways.iu.edu:18880/api/v1/agent/executejupyterrequest'
+ url =
'https://api.gateway.cybershuttle.org/api/v1/agent/executejupyterrequest'
data = {
"sessionId": "session1",
@@ -149,7 +149,7 @@ def run_remote(line, cell):
print("Cell execution failed. Error: " + error)
if execution_id:
while True:
- url =
"http://scigap02.sciencegateways.iu.edu:18880/api/v1/agent/executejupyterresponse/";
+ execution_id
+ url =
"https://api.gateway.cybershuttle.org/api/v1/agent/executejupyterresponse/"; +
execution_id
response = requests.get(url, headers={'Accept':
'application/json'})
json_response = response.json()
#print(json_response)
